[X86] VRNDSCALE* folding from masked and scalar ffloor and fceil patterns

[mike.dvoretsky]

Summary

This patch handles back-end folding of generic patterns created by D48067 in cases where the instruction isn't a straightforward packed values rounding operation, but a masked operation or a scalar operation.

[craig.topper]

Tests?

[mike.dvoretsky]

Added tests for folding. Also made changes to enable the new version of D45202.

[craig.topper]

llvm/lib/Target/X86/X86ISelLowering.cpp

30964

Can we just do this with isel patterns like we do for ADDSS?

llvm/test/CodeGen/X86/vec_floor.ll

874

Can you generate %k from a compare instruction rather than passing in a X x i1 type. It will make the code a little cleaner since we won't have to extend and split the mask in such crazy ways.

[mike.dvoretsky]

llvm/lib/Target/X86/X86ISelLowering.cpp

30964

I've considered that, but decided to fold it here. To do it in .td patterns we'd need to add 4 new patterns in 2 separate files. 32 and 64 bit patterns would need to be added for VROUNDS* on AVX and ROUNDS* on SSE4.1. Writing this pattern here both makes it easier to track and produces less check complexity.

[craig.topper]

llvm/lib/Target/X86/X86ISelLowering.cpp

30958

There's a signed vs unsigned comparison warning on this line.

[mike.dvoretsky]

Changed the folding to use isel patterns from D47012.

[craig.topper]

llvm/lib/Target/X86/X86InstrAVX512.td

8736

Do we have test cases covering this pattern? I can't find any zero extend instructions

[mike.dvoretsky]

Added zero extension of mask to i32 in the masked scalar tests and added more ways to represent the mask, testing the 8-bit mask pattern among others. 16-bit mask patterns removed due to scalar_to_vector errors.

[craig.topper]

llvm/lib/Target/X86/X86InstrAVX512.td

8735

Why HasVLX? Shouldn't scalar instructions be valid under HasAVX512?

9912

What about zero masking?

[mike.dvoretsky]

Corrected the scalar pattern predicates, added packed zero-masked instruction patterns and tests to cover zero-masking. Changed the RUN line of vec_floor.ll to give different results for AVX512F and AVX512VL where needed (e.g. in 128- and 256-bit masked operations).

[mike.dvoretsky]

Added tests for floor intrinsics and masked scalar double patterns to cover all introduced isel patterns.

[craig.topper]

LGTM

[mike.dvoretsky]

Updated to support D48067 instead of D45202. Removed IntrinsicsX86.td changes since the lowering is now in InstCombine, changed the masked scalar pattern tests to exclude mask extension that was in the previous versions.

[mike.dvoretsky]

Updated the AVX512 isel pattern to account for the instruction record name change.

[craig.topper]

LGTM

llvm/lib/Target/X86/X86ISelLowering.cpp

static SDValue combineShuffleOfConcatUndef(SDNode *N, SelectionDAG &DAG,
   if (VT.getVectorElementType() != MVT::i32 &&
       VT.getVectorElementType() != MVT::i64 &&
       VT.getVectorElementType() != MVT::f32 &&
       VT.getVectorElementType() != MVT::f64)
     return SDValue();
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
 
   // Check that both sources are concats with undef.
   if (N0.getOpcode() != ISD::CONCAT_VECTORS ||
       N1.getOpcode() != ISD::CONCAT_VECTORS || N0.getNumOperands() != 2 ||
       N1.getNumOperands() != 2 || !N0.getOperand(1).isUndef() ||
       !N1.getOperand(1).isUndef())
     return SDValue();
 
   // Construct the new shuffle mask. Elements from the first source retain their
   // index, but elements from the second source no longer need to skip an undef.
   SmallVector<int, 8> Mask;
   int NumElts = VT.getVectorNumElements();
 
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
   for (int Elt : SVOp->getMask())
static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG) {
   SDValue Src = N->getOperand(0);
 
   // If this is a scalar to vector to v1i1 from an AND with 1, bypass the and.
   // This occurs frequently in our masked scalar intrinsic code and our
   // floating point select lowering with AVX512.
   // TODO: SimplifyDemandedBits instead?
   if (VT == MVT::v1i1 && Src.getOpcode() == ISD::AND && Src.hasOneUse())
     if (auto *C = dyn_cast<ConstantSDNode>(Src.getOperand(1)))
-      if (C->getAPIntValue().isOneValue())
-        return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), MVT::v1i1,
-                           Src.getOperand(0));
+      if (C->getAPIntValue().isOneValue()) {
+        SDValue Mask = Src.getOperand(0);
+        if (Mask.getOpcode() == ISD::TRUNCATE &&
+            Mask.getOperand(0).getValueType() != MVT::i16)
+          Mask = Mask.getOperand(0);
+        return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), MVT::v1i1, Mask);
+      }
+
+  // The result of AND may also be truncated. This occurs in code for lowered
+  // masked scalar intrinsics.
+  if (VT == MVT::v1i1 && Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse() &&
+      Src.getOperand(0).getOpcode() == ISD::AND &&
+      Src.getOperand(0).hasOneUse())
+    if (auto *C = dyn_cast<ConstantSDNode>(Src.getOperand(0).getOperand(1)))
+      if (C->getAPIntValue().isOneValue()) {
+        SDValue Mask = Src.getOperand(0).getOperand(0);
+        if (Mask.getOpcode() == ISD::TRUNCATE &&
+            Mask.getOperand(0).getValueType() != MVT::i16)
+          Mask = Mask.getOperand(0);
+        // Check if the initial value is an i16. scalar_to_vector fails to
+        // select for that type, so the combine should be aborted.
+        if (Mask.getValueType() == MVT::i16)
+          return SDValue();
+        return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), MVT::v1i1, Mask);
+      }
 
   return SDValue();
 }
 
 // Simplify PMULDQ and PMULUDQ operations.
 static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
                              TargetLowering::DAGCombinerInfo &DCI) {
   SDValue LHS = N->getOperand(0);

llvm/lib/Target/X86/X86InstrAVX512.td

let Predicates = [BasePredicate] in {
 
     def : Pat<(Move _.VT:$src1, (scalar_to_vector (X86selects Mask,
                (OpNode (extractelt _.VT:$src2, (iPTR 0))), ZeroFP))),
               (!cast<Instruction>("V"#OpcPrefix#r_Intkz)
                OutMask, _.VT:$src1, _.VT:$src2, (i32 ImmV))>;
   }
 }
 
+defm : avx512_masked_scalar_imm<ffloor, "RNDSCALESS", X86Movss,
+                                (v1i1 (scalar_to_vector GR32:$mask)),
+                                v4f32x_info, fp32imm0, 0x01,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
+defm : avx512_masked_scalar_imm<ffloor, "RNDSCALESS", X86Movss,
+                                (v1i1 (scalar_to_vector GR8:$mask)),
+                                v4f32x_info, fp32imm0, 0x01,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
+defm : avx512_masked_scalar_imm<fceil, "RNDSCALESS", X86Movss,
+                                (v1i1 (scalar_to_vector GR32:$mask)),
+                                v4f32x_info, fp32imm0, 0x02,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
+defm : avx512_masked_scalar_imm<fceil, "RNDSCALESS", X86Movss,
+                                (v1i1 (scalar_to_vector GR8:$mask)),
+                                v4f32x_info, fp32imm0, 0x02,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
+defm : avx512_masked_scalar_imm<ffloor, "RNDSCALESD", X86Movsd,
+                                (v1i1 (scalar_to_vector GR32:$mask)),
+                                v2f64x_info, fp64imm0, 0x01,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
+defm : avx512_masked_scalar_imm<ffloor, "RNDSCALESD", X86Movsd,
+                                (v1i1 (scalar_to_vector GR8:$mask)),
+                                v2f64x_info, fp64imm0, 0x01,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
+defm : avx512_masked_scalar_imm<fceil, "RNDSCALESD", X86Movsd,
+                                (v1i1 (scalar_to_vector GR32:$mask)),
+                                v2f64x_info, fp64imm0, 0x02,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
+defm : avx512_masked_scalar_imm<fceil, "RNDSCALESD", X86Movsd,
+                                (v1i1 (scalar_to_vector GR8:$mask)),
+                                v2f64x_info, fp64imm0, 0x02,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
+
+
 //-------------------------------------------------
 // Integer truncate and extend operations
 //-------------------------------------------------
 
 multiclass avx512_trunc_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
                               X86FoldableSchedWrite sched, X86VectorVTInfo SrcInfo,
                               X86VectorVTInfo DestInfo, X86MemOperand x86memop> {
   let ExeDomain = DestInfo.ExeDomain in
defm VGETMANTSD: avx512_common_fp_sae_scalar_imm<"vgetmantsd", f64x_info,
       AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
 defm VGETMANTSS: avx512_common_fp_sae_scalar_imm<"vgetmantss", f32x_info,
       0x27, X86GetMants, X86GetMantsRnd, SchedWriteFRnd, HasAVX512>,
       AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
 
 let Predicates = [HasAVX512] in {
 def : Pat<(v16f32 (ffloor VR512:$src)),
           (VRNDSCALEPSZrri VR512:$src, (i32 0x9))>;
+def : Pat<(v16f32 (vselect VK16WM:$mask, (ffloor VR512:$src), VR512:$dst)),
+          (VRNDSCALEPSZrrik VR512:$dst, VK16WM:$mask, VR512:$src, (i32 0x9))>;
+def : Pat<(v16f32 (vselect VK16WM:$mask, (ffloor VR512:$src), v16f32_info.ImmAllZerosV)),
+          (VRNDSCALEPSZrrikz VK16WM:$mask, VR512:$src, (i32 0x9))>;
 def : Pat<(v16f32 (fnearbyint VR512:$src)),
           (VRNDSCALEPSZrri VR512:$src, (i32 0xC))>;
 def : Pat<(v16f32 (fceil VR512:$src)),
           (VRNDSCALEPSZrri VR512:$src, (i32 0xA))>;
+def : Pat<(v16f32 (vselect VK16WM:$mask, (fceil VR512:$src), VR512:$dst)),
+          (VRNDSCALEPSZrrik VR512:$dst, VK16WM:$mask, VR512:$src, (i32 0xA))>;
+def : Pat<(v16f32 (vselect VK16WM:$mask, (fceil VR512:$src), v16f32_info.ImmAllZerosV)),
+          (VRNDSCALEPSZrrikz VK16WM:$mask, VR512:$src, (i32 0xA))>;
 def : Pat<(v16f32 (frint VR512:$src)),
           (VRNDSCALEPSZrri VR512:$src, (i32 0x4))>;
 def : Pat<(v16f32 (ftrunc VR512:$src)),
           (VRNDSCALEPSZrri VR512:$src, (i32 0xB))>;
 
 def : Pat<(v16f32 (ffloor (loadv16f32 addr:$src))),
           (VRNDSCALEPSZrmi addr:$src, (i32 0x9))>;
 def : Pat<(v16f32 (fnearbyint (loadv16f32 addr:$src))),
           (VRNDSCALEPSZrmi addr:$src, (i32 0xC))>;
 def : Pat<(v16f32 (fceil (loadv16f32 addr:$src))),
           (VRNDSCALEPSZrmi addr:$src, (i32 0xA))>;
 def : Pat<(v16f32 (frint (loadv16f32 addr:$src))),
           (VRNDSCALEPSZrmi addr:$src, (i32 0x4))>;
 def : Pat<(v16f32 (ftrunc (loadv16f32 addr:$src))),
           (VRNDSCALEPSZrmi addr:$src, (i32 0xB))>;
 
 def : Pat<(v8f64 (ffloor VR512:$src)),
           (VRNDSCALEPDZrri VR512:$src, (i32 0x9))>;
+def : Pat<(v8f64 (vselect VK8WM:$mask, (ffloor VR512:$src), VR512:$dst)),
+          (VRNDSCALEPDZrrik VR512:$dst, VK8WM:$mask, VR512:$src, (i32 0x9))>;
+def : Pat<(v8f64 (vselect VK8WM:$mask, (ffloor VR512:$src), v8f64_info.ImmAllZerosV)),
+          (VRNDSCALEPDZrrikz VK8WM:$mask, VR512:$src, (i32 0x9))>;
 def : Pat<(v8f64 (fnearbyint VR512:$src)),
           (VRNDSCALEPDZrri VR512:$src, (i32 0xC))>;
 def : Pat<(v8f64 (fceil VR512:$src)),
           (VRNDSCALEPDZrri VR512:$src, (i32 0xA))>;
+def : Pat<(v8f64 (vselect VK8WM:$mask, (fceil VR512:$src), VR512:$dst)),
+          (VRNDSCALEPDZrrik VR512:$dst, VK8WM:$mask, VR512:$src, (i32 0xA))>;
+def : Pat<(v8f64 (vselect VK8WM:$mask, (fceil VR512:$src), v8f64_info.ImmAllZerosV)),
+          (VRNDSCALEPDZrrikz VK8WM:$mask, VR512:$src, (i32 0xA))>;
 def : Pat<(v8f64 (frint VR512:$src)),
           (VRNDSCALEPDZrri VR512:$src, (i32 0x4))>;
 def : Pat<(v8f64 (ftrunc VR512:$src)),
           (VRNDSCALEPDZrri VR512:$src, (i32 0xB))>;
 
 def : Pat<(v8f64 (ffloor (loadv8f64 addr:$src))),
           (VRNDSCALEPDZrmi addr:$src, (i32 0x9))>;
 def : Pat<(v8f64 (fnearbyint (loadv8f64 addr:$src))),
           (VRNDSCALEPDZrmi addr:$src, (i32 0xC))>;
 def : Pat<(v8f64 (fceil (loadv8f64 addr:$src))),
           (VRNDSCALEPDZrmi addr:$src, (i32 0xA))>;
 def : Pat<(v8f64 (frint (loadv8f64 addr:$src))),
           (VRNDSCALEPDZrmi addr:$src, (i32 0x4))>;
 def : Pat<(v8f64 (ftrunc (loadv8f64 addr:$src))),
           (VRNDSCALEPDZrmi addr:$src, (i32 0xB))>;
 }
 
 let Predicates = [HasVLX] in {
 def : Pat<(v4f32 (ffloor VR128X:$src)),
           (VRNDSCALEPSZ128rri VR128X:$src, (i32 0x9))>;
+def : Pat<(v4f32 (vselect VK4WM:$mask, (ffloor VR128X:$src), VR128X:$dst)),
+          (VRNDSCALEPSZ128rrik VR128X:$dst, VK4WM:$mask, VR128X:$src, (i32 0x9))>;
+def : Pat<(v4f32 (vselect VK4WM:$mask, (ffloor VR128X:$src), v4f32x_info.ImmAllZerosV)),
+          (VRNDSCALEPSZ128rrikz VK4WM:$mask, VR128X:$src, (i32 0x9))>;
 def : Pat<(v4f32 (fnearbyint VR128X:$src)),
           (VRNDSCALEPSZ128rri VR128X:$src, (i32 0xC))>;
 def : Pat<(v4f32 (fceil VR128X:$src)),
           (VRNDSCALEPSZ128rri VR128X:$src, (i32 0xA))>;
+def : Pat<(v4f32 (vselect VK4WM:$mask, (fceil VR128X:$src), VR128X:$dst)),
+          (VRNDSCALEPSZ128rrik VR128X:$dst, VK4WM:$mask, VR128X:$src, (i32 0xA))>;
+def : Pat<(v4f32 (vselect VK4WM:$mask, (fceil VR128X:$src), v4f32x_info.ImmAllZerosV)),
+          (VRNDSCALEPSZ128rrikz VK4WM:$mask, VR128X:$src, (i32 0xA))>;
 def : Pat<(v4f32 (frint VR128X:$src)),
           (VRNDSCALEPSZ128rri VR128X:$src, (i32 0x4))>;
 def : Pat<(v4f32 (ftrunc VR128X:$src)),
           (VRNDSCALEPSZ128rri VR128X:$src, (i32 0xB))>;
 
 def : Pat<(v4f32 (ffloor (loadv4f32 addr:$src))),
           (VRNDSCALEPSZ128rmi addr:$src, (i32 0x9))>;
 def : Pat<(v4f32 (fnearbyint (loadv4f32 addr:$src))),
           (VRNDSCALEPSZ128rmi addr:$src, (i32 0xC))>;
 def : Pat<(v4f32 (fceil (loadv4f32 addr:$src))),
           (VRNDSCALEPSZ128rmi addr:$src, (i32 0xA))>;
 def : Pat<(v4f32 (frint (loadv4f32 addr:$src))),
           (VRNDSCALEPSZ128rmi addr:$src, (i32 0x4))>;
 def : Pat<(v4f32 (ftrunc (loadv4f32 addr:$src))),
           (VRNDSCALEPSZ128rmi addr:$src, (i32 0xB))>;
 
 def : Pat<(v2f64 (ffloor VR128X:$src)),
           (VRNDSCALEPDZ128rri VR128X:$src, (i32 0x9))>;
+def : Pat<(v2f64 (vselect VK2WM:$mask, (ffloor VR128X:$src), VR128X:$dst)),
+          (VRNDSCALEPDZ128rrik VR128X:$dst, VK2WM:$mask, VR128X:$src, (i32 0x9))>;
+def : Pat<(v2f64 (vselect VK2WM:$mask, (ffloor VR128X:$src), v2f64x_info.ImmAllZerosV)),
+          (VRNDSCALEPDZ128rrikz VK2WM:$mask, VR128X:$src, (i32 0x9))>;
 def : Pat<(v2f64 (fnearbyint VR128X:$src)),
           (VRNDSCALEPDZ128rri VR128X:$src, (i32 0xC))>;
 def : Pat<(v2f64 (fceil VR128X:$src)),
           (VRNDSCALEPDZ128rri VR128X:$src, (i32 0xA))>;
+def : Pat<(v2f64 (vselect VK2WM:$mask, (fceil VR128X:$src), VR128X:$dst)),
+          (VRNDSCALEPDZ128rrik VR128X:$dst, VK2WM:$mask, VR128X:$src, (i32 0xA))>;
+def : Pat<(v2f64 (vselect VK2WM:$mask, (fceil VR128X:$src), v2f64x_info.ImmAllZerosV)),
+          (VRNDSCALEPDZ128rrikz VK2WM:$mask, VR128X:$src, (i32 0xA))>;
 def : Pat<(v2f64 (frint VR128X:$src)),
           (VRNDSCALEPDZ128rri VR128X:$src, (i32 0x4))>;
 def : Pat<(v2f64 (ftrunc VR128X:$src)),
           (VRNDSCALEPDZ128rri VR128X:$src, (i32 0xB))>;
 
 def : Pat<(v2f64 (ffloor (loadv2f64 addr:$src))),
           (VRNDSCALEPDZ128rmi addr:$src, (i32 0x9))>;
 def : Pat<(v2f64 (fnearbyint (loadv2f64 addr:$src))),
           (VRNDSCALEPDZ128rmi addr:$src, (i32 0xC))>;
 def : Pat<(v2f64 (fceil (loadv2f64 addr:$src))),
           (VRNDSCALEPDZ128rmi addr:$src, (i32 0xA))>;
 def : Pat<(v2f64 (frint (loadv2f64 addr:$src))),
           (VRNDSCALEPDZ128rmi addr:$src, (i32 0x4))>;
 def : Pat<(v2f64 (ftrunc (loadv2f64 addr:$src))),
           (VRNDSCALEPDZ128rmi addr:$src, (i32 0xB))>;
 
 def : Pat<(v8f32 (ffloor VR256X:$src)),
           (VRNDSCALEPSZ256rri VR256X:$src, (i32 0x9))>;
+def : Pat<(v8f32 (vselect VK8WM:$mask, (ffloor VR256X:$src), VR256X:$dst)),
+          (VRNDSCALEPSZ256rrik VR256X:$dst, VK8WM:$mask, VR256X:$src, (i32 0x9))>;
+def : Pat<(v8f32 (vselect VK8WM:$mask, (ffloor VR256X:$src), v8f32x_info.ImmAllZerosV)),
+          (VRNDSCALEPSZ256rrikz VK8WM:$mask, VR256X:$src, (i32 0x9))>;
 def : Pat<(v8f32 (fnearbyint VR256X:$src)),
           (VRNDSCALEPSZ256rri VR256X:$src, (i32 0xC))>;
 def : Pat<(v8f32 (fceil VR256X:$src)),
           (VRNDSCALEPSZ256rri VR256X:$src, (i32 0xA))>;
+def : Pat<(v8f32 (vselect VK8WM:$mask, (fceil VR256X:$src), VR256X:$dst)),
+          (VRNDSCALEPSZ256rrik VR256X:$dst, VK8WM:$mask, VR256X:$src, (i32 0xA))>;
+def : Pat<(v8f32 (vselect VK8WM:$mask, (fceil VR256X:$src), v8f32x_info.ImmAllZerosV)),
+          (VRNDSCALEPSZ256rrikz VK8WM:$mask, VR256X:$src, (i32 0xA))>;
 def : Pat<(v8f32 (frint VR256X:$src)),
           (VRNDSCALEPSZ256rri VR256X:$src, (i32 0x4))>;
 def : Pat<(v8f32 (ftrunc VR256X:$src)),
           (VRNDSCALEPSZ256rri VR256X:$src, (i32 0xB))>;
 
 def : Pat<(v8f32 (ffloor (loadv8f32 addr:$src))),
           (VRNDSCALEPSZ256rmi addr:$src, (i32 0x9))>;
 def : Pat<(v8f32 (fnearbyint (loadv8f32 addr:$src))),
           (VRNDSCALEPSZ256rmi addr:$src, (i32 0xC))>;
 def : Pat<(v8f32 (fceil (loadv8f32 addr:$src))),
           (VRNDSCALEPSZ256rmi addr:$src, (i32 0xA))>;
 def : Pat<(v8f32 (frint (loadv8f32 addr:$src))),
           (VRNDSCALEPSZ256rmi addr:$src, (i32 0x4))>;
 def : Pat<(v8f32 (ftrunc (loadv8f32 addr:$src))),
           (VRNDSCALEPSZ256rmi addr:$src, (i32 0xB))>;
 
 def : Pat<(v4f64 (ffloor VR256X:$src)),
           (VRNDSCALEPDZ256rri VR256X:$src, (i32 0x9))>;
+def : Pat<(v4f64 (vselect VK4WM:$mask, (ffloor VR256X:$src), VR256X:$dst)),
+          (VRNDSCALEPDZ256rrik VR256X:$dst, VK4WM:$mask, VR256X:$src, (i32 0x9))>;
+def : Pat<(v4f64 (vselect VK4WM:$mask, (ffloor VR256X:$src), v4f64x_info.ImmAllZerosV)),
+          (VRNDSCALEPDZ256rrikz VK4WM:$mask, VR256X:$src, (i32 0x9))>;
 def : Pat<(v4f64 (fnearbyint VR256X:$src)),
           (VRNDSCALEPDZ256rri VR256X:$src, (i32 0xC))>;
 def : Pat<(v4f64 (fceil VR256X:$src)),
           (VRNDSCALEPDZ256rri VR256X:$src, (i32 0xA))>;
+def : Pat<(v4f64 (vselect VK4WM:$mask, (fceil VR256X:$src), VR256X:$dst)),
+          (VRNDSCALEPDZ256rrik VR256X:$dst, VK4WM:$mask, VR256X:$src, (i32 0xA))>;
+def : Pat<(v4f64 (vselect VK4WM:$mask, (fceil VR256X:$src), v4f64x_info.ImmAllZerosV)),
+          (VRNDSCALEPDZ256rrikz VK4WM:$mask, VR256X:$src, (i32 0xA))>;
 def : Pat<(v4f64 (frint VR256X:$src)),
           (VRNDSCALEPDZ256rri VR256X:$src, (i32 0x4))>;
 def : Pat<(v4f64 (ftrunc VR256X:$src)),
           (VRNDSCALEPDZ256rri VR256X:$src, (i32 0xB))>;
 
 def : Pat<(v4f64 (ffloor (loadv4f64 addr:$src))),
           (VRNDSCALEPDZ256rmi addr:$src, (i32 0x9))>;
 def : Pat<(v4f64 (fnearbyint (loadv4f64 addr:$src))),

llvm/lib/Target/X86/X86InstrSSE.td

let Predicates = [UseSSE41] in {
   def : Pat<(v2f64 (fceil (loadv2f64 addr:$src))),
             (ROUNDPDm addr:$src, (i32 0xA))>;
   def : Pat<(v2f64 (frint (loadv2f64 addr:$src))),
             (ROUNDPDm addr:$src, (i32 0x4))>;
   def : Pat<(v2f64 (ftrunc (loadv2f64 addr:$src))),
             (ROUNDPDm addr:$src, (i32 0xB))>;
 }
 
+defm : scalar_unary_math_imm_patterns<ffloor, "ROUNDSS", X86Movss,
+                                      v4f32, 0x01, UseSSE41>;
+defm : scalar_unary_math_imm_patterns<fceil, "ROUNDSS", X86Movss,
+                                      v4f32, 0x02, UseSSE41>;
+defm : scalar_unary_math_imm_patterns<ffloor, "ROUNDSD", X86Movsd,
+                                      v2f64, 0x01, UseSSE41>;
+defm : scalar_unary_math_imm_patterns<fceil, "ROUNDSD", X86Movsd,
+                                      v2f64, 0x02, UseSSE41>;
+
 //===----------------------------------------------------------------------===//
 // SSE4.1 - Packed Bit Test
 //===----------------------------------------------------------------------===//
 
 // ptest instruction we'll lower to this in X86ISelLowering primarily from
 // the intel intrinsic that corresponds to this.
 let Defs = [EFLAGS], Predicates = [HasAVX] in {
 def VPTESTrr  : SS48I<0x17, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src2),

llvm/test/CodeGen/X86/vec_floor.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE41
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx | FileCheck %s --check-prefix=AVX
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512vl | FileCheck %s --check-prefix=AVX512
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f | FileCheck %s --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512vl | FileCheck %s --check-prefix=AVX512 --check-prefix=AVX512VL
 
 define <2 x double> @floor_v2f64(<2 x double> %p) {
 ; SSE41-LABEL: floor_v2f64:
 ; SSE41:       ## %bb.0:
 ; SSE41-NEXT:    roundpd $9, %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: floor_v2f64:

 ;
 ; AVX512-LABEL: const_trunc_v4f32:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vmovaps {{.*#+}} xmm0 = [-3.000000e+00,6.000000e+00,-9.000000e+00,2.000000e+00]
 ; AVX512-NEXT:    retq
   %t = call <4 x float> @llvm.trunc.v4f32(<4 x float> <float -3.5, float 6.0, float -9.0, float 2.5>)
   ret <4 x float> %t
 }
+
+;
+; Scalar and masked instructions
+;
+
+define <4 x float> @floor_ss(<4 x float> %x, <4 x float> %y) nounwind {
+; SSE41-LABEL: floor_ss:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundss $1, %xmm0, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_ss:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vroundss $1, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: floor_ss:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vroundss $1, %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %s = extractelement <4 x float> %x, i32 0
+  %call = call float @llvm.floor.f32(float %s)
+  %res = insertelement <4 x float> %y, float %call, i32 0
+  ret <4 x float> %res
+}
+declare float @llvm.floor.f32(float %s)
+
+define <2 x double> @floor_sd(<2 x double> %x, <2 x double> %y) nounwind {
+; SSE41-LABEL: floor_sd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundsd $1, %xmm0, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_sd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vroundsd $1, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: floor_sd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vroundsd $1, %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %s = extractelement <2 x double> %x, i32 0
+  %call = call double @llvm.floor.f64(double %s)
+  %res = insertelement <2 x double> %y, double %call, i32 0
+  ret <2 x double> %res
+}
+declare double @llvm.floor.f64(double %s)
+
+define <4 x float> @floor_mask_128_ps(<4 x float> %x, <4 x float> %y) nounwind {
+; SSE41-LABEL: floor_mask_128_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundps $9, %xmm0, %xmm2
+; SSE41-NEXT:    cmpeqps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_mask_128_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vroundps $9, %xmm0, %xmm0
+; AVX-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: floor_mask_128_ps:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundps $9, %xmm0, %xmm0
+; AVX512F-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: floor_mask_128_ps:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqps %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscaleps $9, %xmm0, %xmm1 {%k1}
+; AVX512VL-NEXT:    vmovaps %xmm1, %xmm0
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <4 x float> %x, %y
+  %call = call <4 x float> @llvm.floor.v4f32(<4 x float> %x)
+  %res = select <4 x i1> %k, <4 x float> %call, <4 x float> %y
+  ret <4 x float> %res
+}
+
+define <4 x float> @floor_maskz_128_ps(<4 x float> %x, <4 x float> %y) nounwind {
+; SSE41-LABEL: floor_maskz_128_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    cmpeqps %xmm0, %xmm1
+; SSE41-NEXT:    roundps $9, %xmm0, %xmm0
+; SSE41-NEXT:    andps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_maskz_128_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm1
+; AVX-NEXT:    vroundps $9, %xmm0, %xmm0
+; AVX-NEXT:    vandps %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: floor_maskz_128_ps:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundps $9, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovaps %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: floor_maskz_128_ps:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqps %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscaleps $9, %xmm0, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <4 x float> %x, %y
+  %call = call <4 x float> @llvm.floor.v4f32(<4 x float> %x)
+  %res = select <4 x i1> %k, <4 x float> %call, <4 x float> zeroinitializer
+  ret <4 x float> %res
+}
+
+define <2 x double> @floor_mask_128_pd(<2 x double> %x, <2 x double> %y) nounwind {
+; SSE41-LABEL: floor_mask_128_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundpd $9, %xmm0, %xmm2
+; SSE41-NEXT:    cmpeqpd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_mask_128_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vroundpd $9, %xmm0, %xmm0
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: floor_mask_128_pd:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundpd $9, %xmm0, %xmm0
+; AVX512F-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: floor_mask_128_pd:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqpd %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscalepd $9, %xmm0, %xmm1 {%k1}
+; AVX512VL-NEXT:    vmovapd %xmm1, %xmm0
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <2 x double> %x, %y
+  %call = call <2 x double> @llvm.floor.v2f64(<2 x double> %x)
+  %res = select <2 x i1> %k, <2 x double> %call, <2 x double> %y
+  ret <2 x double> %res
+}
+
+define <2 x double> @floor_maskz_128_pd(<2 x double> %x, <2 x double> %y) nounwind {
+; SSE41-LABEL: floor_maskz_128_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    cmpeqpd %xmm0, %xmm1
+; SSE41-NEXT:    roundpd $9, %xmm0, %xmm0
+; SSE41-NEXT:    andpd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_maskz_128_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm1
+; AVX-NEXT:    vroundpd $9, %xmm0, %xmm0
+; AVX-NEXT:    vandpd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: floor_maskz_128_pd:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundpd $9, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovapd %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: floor_maskz_128_pd:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqpd %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscalepd $9, %xmm0, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <2 x double> %x, %y
+  %call = call <2 x double> @llvm.floor.v2f64(<2 x double> %x)
+  %res = select <2 x i1> %k, <2 x double> %call, <2 x double> zeroinitializer
+  ret <2 x double> %res
+}
+
+define <8 x float> @floor_mask_256_ps(<8 x float> %x, <8 x float> %y) nounwind {
+; SSE41-LABEL: floor_mask_256_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundps $9, %xmm1, %xmm4
+; SSE41-NEXT:    cmpeqps %xmm3, %xmm1
+; SSE41-NEXT:    roundps $9, %xmm0, %xmm5
+; SSE41-NEXT:    cmpeqps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    movaps %xmm3, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_mask_256_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %ymm1, %ymm0, %ymm2
+; AVX-NEXT:    vroundps $9, %ymm0, %ymm0
+; AVX-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: floor_mask_256_ps:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundps $9, %ymm0, %ymm0
+; AVX512F-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: floor_mask_256_ps:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqps %ymm1, %ymm0, %k1
+; AVX512VL-NEXT:    vrndscaleps $9, %ymm0, %ymm1 {%k1}
+; AVX512VL-NEXT:    vmovaps %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <8 x float> %x, %y
+  %call = call <8 x float> @llvm.floor.v8f32(<8 x float> %x)
+  %res = select <8 x i1> %k, <8 x float> %call, <8 x float> %y
+  ret <8 x float> %res
+}
+
+define <8 x float> @floor_maskz_256_ps(<8 x float> %x, <8 x float> %y) nounwind {
+; SSE41-LABEL: floor_maskz_256_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    cmpeqps %xmm1, %xmm3
+; SSE41-NEXT:    cmpeqps %xmm0, %xmm2
+; SSE41-NEXT:    roundps $9, %xmm1, %xmm1
+; SSE41-NEXT:    andps %xmm3, %xmm1
+; SSE41-NEXT:    roundps $9, %xmm0, %xmm0
+; SSE41-NEXT:    andps %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_maskz_256_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %ymm1, %ymm0, %ymm1
+; AVX-NEXT:    vroundps $9, %ymm0, %ymm0
+; AVX-NEXT:    vandps %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: floor_maskz_256_ps:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundps $9, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovaps %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: floor_maskz_256_ps:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqps %ymm1, %ymm0, %k1
+; AVX512VL-NEXT:    vrndscaleps $9, %ymm0, %ymm0 {%k1} {z}
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <8 x float> %x, %y
+  %call = call <8 x float> @llvm.floor.v8f32(<8 x float> %x)
+  %res = select <8 x i1> %k, <8 x float> %call, <8 x float> zeroinitializer
+  ret <8 x float> %res
+}
+
+define <4 x double> @floor_mask_256_pd(<4 x double> %x, <4 x double> %y) nounwind {
+; SSE41-LABEL: floor_mask_256_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundpd $9, %xmm1, %xmm4
+; SSE41-NEXT:    cmpeqpd %xmm3, %xmm1
+; SSE41-NEXT:    roundpd $9, %xmm0, %xmm5
+; SSE41-NEXT:    cmpeqpd %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_mask_256_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm2
+; AVX-NEXT:    vroundpd $9, %ymm0, %ymm0
+; AVX-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: floor_mask_256_pd:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundpd $9, %ymm0, %ymm0
+; AVX512F-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: floor_mask_256_pd:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqpd %ymm1, %ymm0, %k1
+; AVX512VL-NEXT:    vrndscalepd $9, %ymm0, %ymm1 {%k1}
+; AVX512VL-NEXT:    vmovapd %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <4 x double> %x, %y
+  %call = call <4 x double> @llvm.floor.v4f64(<4 x double> %x)
+  %res = select <4 x i1> %k, <4 x double> %call, <4 x double> %y
+  ret <4 x double> %res
+}
+
+define <4 x double> @floor_maskz_256_pd(<4 x double> %x, <4 x double> %y) nounwind {
+; SSE41-LABEL: floor_maskz_256_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    cmpeqpd %xmm1, %xmm3
+; SSE41-NEXT:    cmpeqpd %xmm0, %xmm2
+; SSE41-NEXT:    roundpd $9, %xmm1, %xmm1
+; SSE41-NEXT:    andpd %xmm3, %xmm1
+; SSE41-NEXT:    roundpd $9, %xmm0, %xmm0
+; SSE41-NEXT:    andpd %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_maskz_256_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm1
+; AVX-NEXT:    vroundpd $9, %ymm0, %ymm0
+; AVX-NEXT:    vandpd %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: floor_maskz_256_pd:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundpd $9, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovapd %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: floor_maskz_256_pd:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqpd %ymm1, %ymm0, %k1
+; AVX512VL-NEXT:    vrndscalepd $9, %ymm0, %ymm0 {%k1} {z}
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <4 x double> %x, %y
+  %call = call <4 x double> @llvm.floor.v4f64(<4 x double> %x)
+  %res = select <4 x i1> %k, <4 x double> %call, <4 x double> zeroinitializer
+  ret <4 x double> %res
+}
+
+define <16 x float> @floor_mask_512_ps(<16 x float> %x, <16 x float> %y) nounwind {
+; SSE41-LABEL: floor_mask_512_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundps $9, %xmm3, %xmm8
+; SSE41-NEXT:    cmpeqps %xmm7, %xmm3
+; SSE41-NEXT:    roundps $9, %xmm2, %xmm9
+; SSE41-NEXT:    cmpeqps %xmm6, %xmm2
+; SSE41-NEXT:    roundps $9, %xmm1, %xmm10
+; SSE41-NEXT:    cmpeqps %xmm5, %xmm1
+; SSE41-NEXT:    roundps $9, %xmm0, %xmm11
+; SSE41-NEXT:    cmpeqps %xmm4, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm4
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm10, %xmm5
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm9, %xmm6
+; SSE41-NEXT:    movaps %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm8, %xmm7
+; SSE41-NEXT:    movaps %xmm4, %xmm0
+; SSE41-NEXT:    movaps %xmm5, %xmm1
+; SSE41-NEXT:    movaps %xmm6, %xmm2
+; SSE41-NEXT:    movaps %xmm7, %xmm3
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_mask_512_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %ymm3, %ymm1, %ymm4
+; AVX-NEXT:    vcmpeqps %ymm2, %ymm0, %ymm5
+; AVX-NEXT:    vroundps $9, %ymm1, %ymm1
+; AVX-NEXT:    vroundps $9, %ymm0, %ymm0
+; AVX-NEXT:    vblendvps %ymm5, %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    vblendvps %ymm4, %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: floor_mask_512_ps:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512-NEXT:    vrndscaleps $9, %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovaps %zmm1, %zmm0
+; AVX512-NEXT:    retq
+  %k = fcmp oeq <16 x float> %x, %y
+  %call = call <16 x float> @llvm.floor.v16f32(<16 x float> %x)
+  %res = select <16 x i1> %k, <16 x float> %call, <16 x float> %y
+  ret <16 x float> %res
+}
+
+define <16 x float> @floor_maskz_512_ps(<16 x float> %x, <16 x float> %y) nounwind {
+; SSE41-LABEL: floor_maskz_512_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    cmpeqps %xmm3, %xmm7
+; SSE41-NEXT:    cmpeqps %xmm2, %xmm6
+; SSE41-NEXT:    cmpeqps %xmm1, %xmm5
+; SSE41-NEXT:    cmpeqps %xmm0, %xmm4
+; SSE41-NEXT:    roundps $9, %xmm3, %xmm3
+; SSE41-NEXT:    andps %xmm7, %xmm3
+; SSE41-NEXT:    roundps $9, %xmm2, %xmm2
+; SSE41-NEXT:    andps %xmm6, %xmm2
+; SSE41-NEXT:    roundps $9, %xmm1, %xmm1
+; SSE41-NEXT:    andps %xmm5, %xmm1
+; SSE41-NEXT:    roundps $9, %xmm0, %xmm0
+; SSE41-NEXT:    andps %xmm4, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_maskz_512_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %ymm3, %ymm1, %ymm3
+; AVX-NEXT:    vcmpeqps %ymm2, %ymm0, %ymm2
+; AVX-NEXT:    vroundps $9, %ymm1, %ymm1
+; AVX-NEXT:    vandps %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    vroundps $9, %ymm0, %ymm0
+; AVX-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: floor_maskz_512_ps:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512-NEXT:    vrndscaleps $9, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %k = fcmp oeq <16 x float> %x, %y
+  %call = call <16 x float> @llvm.floor.v16f32(<16 x float> %x)
+  %res = select <16 x i1> %k, <16 x float> %call, <16 x float> zeroinitializer
+  ret <16 x float> %res
+}
+
+define <8 x double> @floor_mask_512_pd(<8 x double> %x, <8 x double> %y) nounwind {
+; SSE41-LABEL: floor_mask_512_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundpd $9, %xmm3, %xmm8
+; SSE41-NEXT:    cmpeqpd %xmm7, %xmm3
+; SSE41-NEXT:    roundpd $9, %xmm2, %xmm9
+; SSE41-NEXT:    cmpeqpd %xmm6, %xmm2
+; SSE41-NEXT:    roundpd $9, %xmm1, %xmm10
+; SSE41-NEXT:    cmpeqpd %xmm5, %xmm1
+; SSE41-NEXT:    roundpd $9, %xmm0, %xmm11
+; SSE41-NEXT:    cmpeqpd %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm4
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm5
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm6
+; SSE41-NEXT:    movapd %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm7
+; SSE41-NEXT:    movapd %xmm4, %xmm0
+; SSE41-NEXT:    movapd %xmm5, %xmm1
+; SSE41-NEXT:    movapd %xmm6, %xmm2
+; SSE41-NEXT:    movapd %xmm7, %xmm3
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_mask_512_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %ymm3, %ymm1, %ymm4
+; AVX-NEXT:    vcmpeqpd %ymm2, %ymm0, %ymm5
+; AVX-NEXT:    vroundpd $9, %ymm1, %ymm1
+; AVX-NEXT:    vroundpd $9, %ymm0, %ymm0
+; AVX-NEXT:    vblendvpd %ymm5, %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    vblendvpd %ymm4, %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: floor_mask_512_pd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512-NEXT:    vrndscalepd $9, %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovapd %zmm1, %zmm0
+; AVX512-NEXT:    retq
+  %k = fcmp oeq <8 x double> %x, %y
+  %call = call <8 x double> @llvm.floor.v8f64(<8 x double> %x)
+  %res = select <8 x i1> %k, <8 x double> %call, <8 x double> %y
+  ret <8 x double> %res
+}
+
+define <8 x double> @floor_maskz_512_pd(<8 x double> %x, <8 x double> %y) nounwind {
+; SSE41-LABEL: floor_maskz_512_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    cmpeqpd %xmm3, %xmm7
+; SSE41-NEXT:    cmpeqpd %xmm2, %xmm6
+; SSE41-NEXT:    cmpeqpd %xmm1, %xmm5
+; SSE41-NEXT:    cmpeqpd %xmm0, %xmm4
+; SSE41-NEXT:    roundpd $9, %xmm3, %xmm3
+; SSE41-NEXT:    andpd %xmm7, %xmm3
+; SSE41-NEXT:    roundpd $9, %xmm2, %xmm2
+; SSE41-NEXT:    andpd %xmm6, %xmm2
+; SSE41-NEXT:    roundpd $9, %xmm1, %xmm1
+; SSE41-NEXT:    andpd %xmm5, %xmm1
+; SSE41-NEXT:    roundpd $9, %xmm0, %xmm0
+; SSE41-NEXT:    andpd %xmm4, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_maskz_512_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %ymm3, %ymm1, %ymm3
+; AVX-NEXT:    vcmpeqpd %ymm2, %ymm0, %ymm2
+; AVX-NEXT:    vroundpd $9, %ymm1, %ymm1
+; AVX-NEXT:    vandpd %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    vroundpd $9, %ymm0, %ymm0
+; AVX-NEXT:    vandpd %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: floor_maskz_512_pd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512-NEXT:    vrndscalepd $9, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %k = fcmp oeq <8 x double> %x, %y
+  %call = call <8 x double> @llvm.floor.v8f64(<8 x double> %x)
+  %res = select <8 x i1> %k, <8 x double> %call, <8 x double> zeroinitializer
+  ret <8 x double> %res
+}
+
+define <4 x float> @floor_mask_ss(<4 x float> %x, <4 x float> %y, <4 x float> %w, i8 %k) nounwind {
+; SSE41-LABEL: floor_mask_ss:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    je LBB50_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundss $9, %xmm0, %xmm2
+; SSE41-NEXT:  LBB50_2:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_mask_ss:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    je LBB50_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB50_2:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: floor_mask_ss:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscaless $1, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512-NEXT:    vmovaps %xmm2, %xmm0
+; AVX512-NEXT:    retq
+  %mask = and i8 %k, 1
+  %nmask = icmp eq i8 %mask, 0
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.floor.f32(float %s)
+  %dst = extractelement <4 x float> %w, i64 0
+  %low = select i1 %nmask, float %dst, float %call
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}
+
+define <4 x float> @floor_maskz_ss(<4 x float> %x, <4 x float> %y, i8 %k) nounwind {
+; SSE41-LABEL: floor_maskz_ss:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    je LBB51_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundss $9, %xmm0, %xmm2
+; SSE41-NEXT:  LBB51_2:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_maskz_ss:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    je LBB51_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB51_2:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: floor_maskz_ss:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscaless $1, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %mask = and i8 %k, 1
+  %nmask = icmp eq i8 %mask, 0
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.floor.f32(float %s)
+  %low = select i1 %nmask, float zeroinitializer, float %call
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}
+
+define <2 x double> @floor_mask_sd(<2 x double> %x, <2 x double> %y, <2 x double> %w, i8 %k) nounwind {
+; SSE41-LABEL: floor_mask_sd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    je LBB52_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundsd $9, %xmm0, %xmm2
+; SSE41-NEXT:  LBB52_2:
+; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_mask_sd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    je LBB52_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB52_2:
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: floor_mask_sd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscalesd $1, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512-NEXT:    vmovapd %xmm2, %xmm0
+; AVX512-NEXT:    retq
+  %mask = and i8 %k, 1
+  %nmask = icmp eq i8 %mask, 0
+  %s = extractelement <2 x double> %x, i64 0
+  %call = tail call double @llvm.floor.f64(double %s)
+  %dst = extractelement <2 x double> %w, i64 0
+  %low = select i1 %nmask, double %dst, double %call
+  %res = insertelement <2 x double> %y, double %low, i64 0
+  ret <2 x double> %res
+}
+
+define <2 x double> @floor_maskz_sd(<2 x double> %x, <2 x double> %y, i8 %k) nounwind {
+; SSE41-LABEL: floor_maskz_sd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    xorpd %xmm2, %xmm2
+; SSE41-NEXT:    je LBB53_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundsd $9, %xmm0, %xmm2
+; SSE41-NEXT:  LBB53_2:
+; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_maskz_sd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    je LBB53_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB53_2:
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: floor_maskz_sd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscalesd $1, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %mask = and i8 %k, 1
+  %nmask = icmp eq i8 %mask, 0
+  %s = extractelement <2 x double> %x, i64 0
+  %call = tail call double @llvm.floor.f64(double %s)
+  %low = select i1 %nmask, double zeroinitializer, double %call
+  %res = insertelement <2 x double> %y, double %low, i64 0
+  ret <2 x double> %res
+}
+
+define <4 x float> @floor_mask_ss_trunc(<4 x float> %x, <4 x float> %y, <4 x float> %w, i16 %k) nounwind {
+; SSE41-LABEL: floor_mask_ss_trunc:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    je LBB54_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundss $9, %xmm0, %xmm2
+; SSE41-NEXT:  LBB54_2:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_mask_ss_trunc:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    je LBB54_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB54_2:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: floor_mask_ss_trunc:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscaless $1, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512-NEXT:    vmovaps %xmm2, %xmm0
+; AVX512-NEXT:    retq
+  %mask = trunc i16 %k to i1
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.floor.f32(float %s)
+  %dst = extractelement <4 x float> %w, i64 0
+  %low = select i1 %mask, float %call, float %dst
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}
+
+define <4 x float> @floor_maskz_ss_trunc(<4 x float> %x, <4 x float> %y, i16 %k) nounwind {
+; SSE41-LABEL: floor_maskz_ss_trunc:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    jne LBB55_1
+; SSE41-NEXT:  ## %bb.2:
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    jmp LBB55_3
+; SSE41-NEXT:  LBB55_1:
+; SSE41-NEXT:    roundss $9, %xmm0, %xmm0
+; SSE41-NEXT:  LBB55_3:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_maskz_ss_trunc:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    jne LBB55_1
+; AVX-NEXT:  ## %bb.2:
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+; AVX-NEXT:  LBB55_1:
+; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: floor_maskz_ss_trunc:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscaless $1, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %mask = trunc i16 %k to i1
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.floor.f32(float %s)
+  %low = select i1 %mask, float %call, float zeroinitializer
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}
+
+define <2 x double> @floor_mask_sd_trunc(<2 x double> %x, <2 x double> %y, <2 x double> %w, i16 %k) nounwind {
+; SSE41-LABEL: floor_mask_sd_trunc:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    je LBB56_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundsd $9, %xmm0, %xmm2
+; SSE41-NEXT:  LBB56_2:
+; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_mask_sd_trunc:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    je LBB56_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB56_2:
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: floor_mask_sd_trunc:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscalesd $1, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512-NEXT:    vmovapd %xmm2, %xmm0
+; AVX512-NEXT:    retq
+  %mask = trunc i16 %k to i1
+  %s = extractelement <2 x double> %x, i64 0
+  %call = tail call double @llvm.floor.f64(double %s)
+  %dst = extractelement <2 x double> %w, i64 0
+  %low = select i1 %mask, double %call, double %dst
+  %res = insertelement <2 x double> %y, double %low, i64 0
+  ret <2 x double> %res
+}
+
+define <2 x double> @floor_maskz_sd_trunc(<2 x double> %x, <2 x double> %y, i16 %k) nounwind {
+; SSE41-LABEL: floor_maskz_sd_trunc:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    jne LBB57_1
+; SSE41-NEXT:  ## %bb.2:
+; SSE41-NEXT:    xorpd %xmm0, %xmm0
+; SSE41-NEXT:    jmp LBB57_3
+; SSE41-NEXT:  LBB57_1:
+; SSE41-NEXT:    roundsd $9, %xmm0, %xmm0
+; SSE41-NEXT:  LBB57_3:
+; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_maskz_sd_trunc:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    jne LBB57_1
+; AVX-NEXT:  ## %bb.2:
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    retq
+; AVX-NEXT:  LBB57_1:
+; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: floor_maskz_sd_trunc:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscalesd $1, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %mask = trunc i16 %k to i1
+  %s = extractelement <2 x double> %x, i64 0
+  %call = tail call double @llvm.floor.f64(double %s)
+  %low = select i1 %mask, double %call, double zeroinitializer
+  %res = insertelement <2 x double> %y, double %low, i64 0
+  ret <2 x double> %res
+}
+
+define <4 x float> @floor_mask_ss_mask8(<4 x float> %x, <4 x float> %y, <4 x float> %w) nounwind {
+; SSE41-LABEL: floor_mask_ss_mask8:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    movaps %xmm0, %xmm3
+; SSE41-NEXT:    cmpeqps %xmm1, %xmm3
+; SSE41-NEXT:    pextrb $0, %xmm3, %eax
+; SSE41-NEXT:    testb $1, %al
+; SSE41-NEXT:    je LBB58_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundss $9, %xmm0, %xmm2
+; SSE41-NEXT:  LBB58_2:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_mask_ss_mask8:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm3
+; AVX-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    je LBB58_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB58_2:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: floor_mask_ss_mask8:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vrndscaless $1, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512F-NEXT:    vmovaps %xmm2, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: floor_mask_ss_mask8:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqps %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscaless $1, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512VL-NEXT:    vmovaps %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
+  %mask1 = fcmp oeq <4 x float> %x, %y
+  %mask = extractelement <4 x i1> %mask1, i64 0
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.floor.f32(float %s)
+  %dst = extractelement <4 x float> %w, i64 0
+  %low = select i1 %mask, float %call, float %dst
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}
+
+define <4 x float> @floor_maskz_ss_mask8(<4 x float> %x, <4 x float> %y) nounwind {
+; SSE41-LABEL: floor_maskz_ss_mask8:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    movaps %xmm0, %xmm2
+; SSE41-NEXT:    cmpeqps %xmm1, %xmm2
+; SSE41-NEXT:    pextrb $0, %xmm2, %eax
+; SSE41-NEXT:    testb $1, %al
+; SSE41-NEXT:    jne LBB59_1
+; SSE41-NEXT:  ## %bb.2:
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    jmp LBB59_3
+; SSE41-NEXT:  LBB59_1:
+; SSE41-NEXT:    roundss $9, %xmm0, %xmm0
+; SSE41-NEXT:  LBB59_3:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_maskz_ss_mask8:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    jne LBB59_1
+; AVX-NEXT:  ## %bb.2:
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+; AVX-NEXT:  LBB59_1:
+; AVX-NEXT:    vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: floor_maskz_ss_mask8:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vrndscaless $1, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: floor_maskz_ss_mask8:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqps %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscaless $1, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    retq
+  %mask1 = fcmp oeq <4 x float> %x, %y
+  %mask = extractelement <4 x i1> %mask1, i64 0
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.floor.f32(float %s)
+  %low = select i1 %mask, float %call, float zeroinitializer
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}
+
+define <2 x double> @floor_mask_sd_mask8(<2 x double> %x, <2 x double> %y, <2 x double> %w) nounwind {
+; SSE41-LABEL: floor_mask_sd_mask8:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    movapd %xmm0, %xmm3
+; SSE41-NEXT:    cmpeqpd %xmm1, %xmm3
+; SSE41-NEXT:    pextrb $0, %xmm3, %eax
+; SSE41-NEXT:    testb $1, %al
+; SSE41-NEXT:    je LBB60_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundsd $9, %xmm0, %xmm2
+; SSE41-NEXT:  LBB60_2:
+; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_mask_sd_mask8:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm3
+; AVX-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    je LBB60_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB60_2:
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: floor_mask_sd_mask8:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vrndscalesd $1, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512F-NEXT:    vmovapd %xmm2, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: floor_mask_sd_mask8:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqpd %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscalesd $1, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512VL-NEXT:    vmovapd %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
+  %mask1 = fcmp oeq <2 x double> %x, %y
+  %mask = extractelement <2 x i1> %mask1, i64 0
+  %s = extractelement <2 x double> %x, i64 0
+  %call = tail call double @llvm.floor.f64(double %s)
+  %dst = extractelement <2 x double> %w, i64 0
+  %low = select i1 %mask, double %call, double %dst
+  %res = insertelement <2 x double> %y, double %low, i64 0
+  ret <2 x double> %res
+}
+
+define <2 x double> @floor_maskz_sd_mask8(<2 x double> %x, <2 x double> %y) nounwind {
+; SSE41-LABEL: floor_maskz_sd_mask8:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    movapd %xmm0, %xmm2
+; SSE41-NEXT:    cmpeqpd %xmm1, %xmm2
+; SSE41-NEXT:    pextrb $0, %xmm2, %eax
+; SSE41-NEXT:    testb $1, %al
+; SSE41-NEXT:    jne LBB61_1
+; SSE41-NEXT:  ## %bb.2:
+; SSE41-NEXT:    xorpd %xmm0, %xmm0
+; SSE41-NEXT:    jmp LBB61_3
+; SSE41-NEXT:  LBB61_1:
+; SSE41-NEXT:    roundsd $9, %xmm0, %xmm0
+; SSE41-NEXT:  LBB61_3:
+; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: floor_maskz_sd_mask8:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    jne LBB61_1
+; AVX-NEXT:  ## %bb.2:
+; AVX-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX-NEXT:    retq
+; AVX-NEXT:  LBB61_1:
+; AVX-NEXT:    vroundsd $9, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: floor_maskz_sd_mask8:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vrndscalesd $1, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: floor_maskz_sd_mask8:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqpd %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscalesd $1, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    retq
+  %mask1 = fcmp oeq <2 x double> %x, %y
+  %mask = extractelement <2 x i1> %mask1, i64 0
+  %s = extractelement <2 x double> %x, i64 0
+  %call = tail call double @llvm.floor.f64(double %s)
+  %low = select i1 %mask, double %call, double zeroinitializer
+  %res = insertelement <2 x double> %y, double %low, i64 0
+  ret <2 x double> %res
+}
+
+define <4 x float> @ceil_ss(<4 x float> %x, <4 x float> %y) nounwind {
+; SSE41-LABEL: ceil_ss:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundss $2, %xmm0, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_ss:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vroundss $2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_ss:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vroundss $2, %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %s = extractelement <4 x float> %x, i32 0
+  %call = call float @llvm.ceil.f32(float %s)
+  %res = insertelement <4 x float> %y, float %call, i32 0
+  ret <4 x float> %res
+}
+declare float @llvm.ceil.f32(float %s)
+
+define <2 x double> @ceil_sd(<2 x double> %x, <2 x double> %y) nounwind {
+; SSE41-LABEL: ceil_sd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundsd $2, %xmm0, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_sd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vroundsd $2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_sd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vroundsd $2, %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %s = extractelement <2 x double> %x, i32 0
+  %call = call double @llvm.ceil.f64(double %s)
+  %res = insertelement <2 x double> %y, double %call, i32 0
+  ret <2 x double> %res
+}
+declare double @llvm.ceil.f64(double %s)
+
+define <4 x float> @ceil_mask_128_ps(<4 x float> %x, <4 x float> %y) nounwind {
+; SSE41-LABEL: ceil_mask_128_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundps $10, %xmm0, %xmm2
+; SSE41-NEXT:    cmpeqps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_128_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vroundps $10, %xmm0, %xmm0
+; AVX-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: ceil_mask_128_ps:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundps $10, %xmm0, %xmm0
+; AVX512F-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: ceil_mask_128_ps:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqps %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscaleps $10, %xmm0, %xmm1 {%k1}
+; AVX512VL-NEXT:    vmovaps %xmm1, %xmm0
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <4 x float> %x, %y
+  %call = call <4 x float> @llvm.ceil.v4f32(<4 x float> %x)
+  %res = select <4 x i1> %k, <4 x float> %call, <4 x float> %y
+  ret <4 x float> %res
+}
+
+define <4 x float> @ceil_maskz_128_ps(<4 x float> %x, <4 x float> %y) nounwind {
+; SSE41-LABEL: ceil_maskz_128_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    cmpeqps %xmm0, %xmm1
+; SSE41-NEXT:    roundps $10, %xmm0, %xmm0
+; SSE41-NEXT:    andps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_maskz_128_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm1
+; AVX-NEXT:    vroundps $10, %xmm0, %xmm0
+; AVX-NEXT:    vandps %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: ceil_maskz_128_ps:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundps $10, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovaps %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: ceil_maskz_128_ps:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqps %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscaleps $10, %xmm0, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <4 x float> %x, %y
+  %call = call <4 x float> @llvm.ceil.v4f32(<4 x float> %x)
+  %res = select <4 x i1> %k, <4 x float> %call, <4 x float> zeroinitializer
+  ret <4 x float> %res
+}
+
+define <2 x double> @ceil_mask_128_pd(<2 x double> %x, <2 x double> %y) nounwind {
+; SSE41-LABEL: ceil_mask_128_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundpd $10, %xmm0, %xmm2
+; SSE41-NEXT:    cmpeqpd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_128_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vroundpd $10, %xmm0, %xmm0
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: ceil_mask_128_pd:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundpd $10, %xmm0, %xmm0
+; AVX512F-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: ceil_mask_128_pd:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqpd %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscalepd $10, %xmm0, %xmm1 {%k1}
+; AVX512VL-NEXT:    vmovapd %xmm1, %xmm0
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <2 x double> %x, %y
+  %call = call <2 x double> @llvm.ceil.v2f64(<2 x double> %x)
+  %res = select <2 x i1> %k, <2 x double> %call, <2 x double> %y
+  ret <2 x double> %res
+}
+
+define <2 x double> @ceil_maskz_128_pd(<2 x double> %x, <2 x double> %y) nounwind {
+; SSE41-LABEL: ceil_maskz_128_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    cmpeqpd %xmm0, %xmm1
+; SSE41-NEXT:    roundpd $10, %xmm0, %xmm0
+; SSE41-NEXT:    andpd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_maskz_128_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm1
+; AVX-NEXT:    vroundpd $10, %xmm0, %xmm0
+; AVX-NEXT:    vandpd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: ceil_maskz_128_pd:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundpd $10, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovapd %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: ceil_maskz_128_pd:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqpd %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscalepd $10, %xmm0, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <2 x double> %x, %y
+  %call = call <2 x double> @llvm.ceil.v2f64(<2 x double> %x)
+  %res = select <2 x i1> %k, <2 x double> %call, <2 x double> zeroinitializer
+  ret <2 x double> %res
+}
+
+define <8 x float> @ceil_mask_256_ps(<8 x float> %x, <8 x float> %y) nounwind {
+; SSE41-LABEL: ceil_mask_256_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundps $10, %xmm1, %xmm4
+; SSE41-NEXT:    cmpeqps %xmm3, %xmm1
+; SSE41-NEXT:    roundps $10, %xmm0, %xmm5
+; SSE41-NEXT:    cmpeqps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    movaps %xmm3, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_256_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %ymm1, %ymm0, %ymm2
+; AVX-NEXT:    vroundps $10, %ymm0, %ymm0
+; AVX-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: ceil_mask_256_ps:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundps $10, %ymm0, %ymm0
+; AVX512F-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: ceil_mask_256_ps:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqps %ymm1, %ymm0, %k1
+; AVX512VL-NEXT:    vrndscaleps $10, %ymm0, %ymm1 {%k1}
+; AVX512VL-NEXT:    vmovaps %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <8 x float> %x, %y
+  %call = call <8 x float> @llvm.ceil.v8f32(<8 x float> %x)
+  %res = select <8 x i1> %k, <8 x float> %call, <8 x float> %y
+  ret <8 x float> %res
+}
+
+define <8 x float> @ceil_maskz_256_ps(<8 x float> %x, <8 x float> %y) nounwind {
+; SSE41-LABEL: ceil_maskz_256_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    cmpeqps %xmm1, %xmm3
+; SSE41-NEXT:    cmpeqps %xmm0, %xmm2
+; SSE41-NEXT:    roundps $10, %xmm1, %xmm1
+; SSE41-NEXT:    andps %xmm3, %xmm1
+; SSE41-NEXT:    roundps $10, %xmm0, %xmm0
+; SSE41-NEXT:    andps %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_maskz_256_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %ymm1, %ymm0, %ymm1
+; AVX-NEXT:    vroundps $10, %ymm0, %ymm0
+; AVX-NEXT:    vandps %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: ceil_maskz_256_ps:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundps $10, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovaps %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: ceil_maskz_256_ps:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqps %ymm1, %ymm0, %k1
+; AVX512VL-NEXT:    vrndscaleps $10, %ymm0, %ymm0 {%k1} {z}
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <8 x float> %x, %y
+  %call = call <8 x float> @llvm.ceil.v8f32(<8 x float> %x)
+  %res = select <8 x i1> %k, <8 x float> %call, <8 x float> zeroinitializer
+  ret <8 x float> %res
+}
+
+define <4 x double> @ceil_mask_256_pd(<4 x double> %x, <4 x double> %y) nounwind {
+; SSE41-LABEL: ceil_mask_256_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundpd $10, %xmm1, %xmm4
+; SSE41-NEXT:    cmpeqpd %xmm3, %xmm1
+; SSE41-NEXT:    roundpd $10, %xmm0, %xmm5
+; SSE41-NEXT:    cmpeqpd %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_256_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm2
+; AVX-NEXT:    vroundpd $10, %ymm0, %ymm0
+; AVX-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: ceil_mask_256_pd:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundpd $10, %ymm0, %ymm0
+; AVX512F-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: ceil_mask_256_pd:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqpd %ymm1, %ymm0, %k1
+; AVX512VL-NEXT:    vrndscalepd $10, %ymm0, %ymm1 {%k1}
+; AVX512VL-NEXT:    vmovapd %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <4 x double> %x, %y
+  %call = call <4 x double> @llvm.ceil.v4f64(<4 x double> %x)
+  %res = select <4 x i1> %k, <4 x double> %call, <4 x double> %y
+  ret <4 x double> %res
+}
+
+define <4 x double> @ceil_maskz_256_pd(<4 x double> %x, <4 x double> %y) nounwind {
+; SSE41-LABEL: ceil_maskz_256_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    cmpeqpd %xmm1, %xmm3
+; SSE41-NEXT:    cmpeqpd %xmm0, %xmm2
+; SSE41-NEXT:    roundpd $10, %xmm1, %xmm1
+; SSE41-NEXT:    andpd %xmm3, %xmm1
+; SSE41-NEXT:    roundpd $10, %xmm0, %xmm0
+; SSE41-NEXT:    andpd %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_maskz_256_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm1
+; AVX-NEXT:    vroundpd $10, %ymm0, %ymm0
+; AVX-NEXT:    vandpd %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: ceil_maskz_256_pd:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vroundpd $10, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovapd %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: ceil_maskz_256_pd:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqpd %ymm1, %ymm0, %k1
+; AVX512VL-NEXT:    vrndscalepd $10, %ymm0, %ymm0 {%k1} {z}
+; AVX512VL-NEXT:    retq
+  %k = fcmp oeq <4 x double> %x, %y
+  %call = call <4 x double> @llvm.ceil.v4f64(<4 x double> %x)
+  %res = select <4 x i1> %k, <4 x double> %call, <4 x double> zeroinitializer
+  ret <4 x double> %res
+}
+
+define <16 x float> @ceil_mask_512_ps(<16 x float> %x, <16 x float> %y) nounwind {
+; SSE41-LABEL: ceil_mask_512_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundps $10, %xmm3, %xmm8
+; SSE41-NEXT:    cmpeqps %xmm7, %xmm3
+; SSE41-NEXT:    roundps $10, %xmm2, %xmm9
+; SSE41-NEXT:    cmpeqps %xmm6, %xmm2
+; SSE41-NEXT:    roundps $10, %xmm1, %xmm10
+; SSE41-NEXT:    cmpeqps %xmm5, %xmm1
+; SSE41-NEXT:    roundps $10, %xmm0, %xmm11
+; SSE41-NEXT:    cmpeqps %xmm4, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm4
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm10, %xmm5
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm9, %xmm6
+; SSE41-NEXT:    movaps %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm8, %xmm7
+; SSE41-NEXT:    movaps %xmm4, %xmm0
+; SSE41-NEXT:    movaps %xmm5, %xmm1
+; SSE41-NEXT:    movaps %xmm6, %xmm2
+; SSE41-NEXT:    movaps %xmm7, %xmm3
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_512_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %ymm3, %ymm1, %ymm4
+; AVX-NEXT:    vcmpeqps %ymm2, %ymm0, %ymm5
+; AVX-NEXT:    vroundps $10, %ymm1, %ymm1
+; AVX-NEXT:    vroundps $10, %ymm0, %ymm0
+; AVX-NEXT:    vblendvps %ymm5, %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    vblendvps %ymm4, %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_512_ps:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512-NEXT:    vrndscaleps $10, %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovaps %zmm1, %zmm0
+; AVX512-NEXT:    retq
+  %k = fcmp oeq <16 x float> %x, %y
+  %call = call <16 x float> @llvm.ceil.v16f32(<16 x float> %x)
+  %res = select <16 x i1> %k, <16 x float> %call, <16 x float> %y
+  ret <16 x float> %res
+}
+
+define <16 x float> @ceil_maskz_512_ps(<16 x float> %x, <16 x float> %y) nounwind {
+; SSE41-LABEL: ceil_maskz_512_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    cmpeqps %xmm3, %xmm7
+; SSE41-NEXT:    cmpeqps %xmm2, %xmm6
+; SSE41-NEXT:    cmpeqps %xmm1, %xmm5
+; SSE41-NEXT:    cmpeqps %xmm0, %xmm4
+; SSE41-NEXT:    roundps $10, %xmm3, %xmm3
+; SSE41-NEXT:    andps %xmm7, %xmm3
+; SSE41-NEXT:    roundps $10, %xmm2, %xmm2
+; SSE41-NEXT:    andps %xmm6, %xmm2
+; SSE41-NEXT:    roundps $10, %xmm1, %xmm1
+; SSE41-NEXT:    andps %xmm5, %xmm1
+; SSE41-NEXT:    roundps $10, %xmm0, %xmm0
+; SSE41-NEXT:    andps %xmm4, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_maskz_512_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %ymm3, %ymm1, %ymm3
+; AVX-NEXT:    vcmpeqps %ymm2, %ymm0, %ymm2
+; AVX-NEXT:    vroundps $10, %ymm1, %ymm1
+; AVX-NEXT:    vandps %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    vroundps $10, %ymm0, %ymm0
+; AVX-NEXT:    vandps %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_maskz_512_ps:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512-NEXT:    vrndscaleps $10, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %k = fcmp oeq <16 x float> %x, %y
+  %call = call <16 x float> @llvm.ceil.v16f32(<16 x float> %x)
+  %res = select <16 x i1> %k, <16 x float> %call, <16 x float> zeroinitializer
+  ret <16 x float> %res
+}
+
+define <8 x double> @ceil_mask_512_pd(<8 x double> %x, <8 x double> %y) nounwind {
+; SSE41-LABEL: ceil_mask_512_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundpd $10, %xmm3, %xmm8
+; SSE41-NEXT:    cmpeqpd %xmm7, %xmm3
+; SSE41-NEXT:    roundpd $10, %xmm2, %xmm9
+; SSE41-NEXT:    cmpeqpd %xmm6, %xmm2
+; SSE41-NEXT:    roundpd $10, %xmm1, %xmm10
+; SSE41-NEXT:    cmpeqpd %xmm5, %xmm1
+; SSE41-NEXT:    roundpd $10, %xmm0, %xmm11
+; SSE41-NEXT:    cmpeqpd %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm4
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm5
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm6
+; SSE41-NEXT:    movapd %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm7
+; SSE41-NEXT:    movapd %xmm4, %xmm0
+; SSE41-NEXT:    movapd %xmm5, %xmm1
+; SSE41-NEXT:    movapd %xmm6, %xmm2
+; SSE41-NEXT:    movapd %xmm7, %xmm3
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_512_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %ymm3, %ymm1, %ymm4
+; AVX-NEXT:    vcmpeqpd %ymm2, %ymm0, %ymm5
+; AVX-NEXT:    vroundpd $10, %ymm1, %ymm1
+; AVX-NEXT:    vroundpd $10, %ymm0, %ymm0
+; AVX-NEXT:    vblendvpd %ymm5, %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    vblendvpd %ymm4, %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_512_pd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512-NEXT:    vrndscalepd $10, %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovapd %zmm1, %zmm0
+; AVX512-NEXT:    retq
+  %k = fcmp oeq <8 x double> %x, %y
+  %call = call <8 x double> @llvm.ceil.v8f64(<8 x double> %x)
+  %res = select <8 x i1> %k, <8 x double> %call, <8 x double> %y
+  ret <8 x double> %res
+}
+
+define <8 x double> @ceil_maskz_512_pd(<8 x double> %x, <8 x double> %y) nounwind {
+; SSE41-LABEL: ceil_maskz_512_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    cmpeqpd %xmm3, %xmm7
+; SSE41-NEXT:    cmpeqpd %xmm2, %xmm6
+; SSE41-NEXT:    cmpeqpd %xmm1, %xmm5
+; SSE41-NEXT:    cmpeqpd %xmm0, %xmm4
+; SSE41-NEXT:    roundpd $10, %xmm3, %xmm3
+; SSE41-NEXT:    andpd %xmm7, %xmm3
+; SSE41-NEXT:    roundpd $10, %xmm2, %xmm2
+; SSE41-NEXT:    andpd %xmm6, %xmm2
+; SSE41-NEXT:    roundpd $10, %xmm1, %xmm1
+; SSE41-NEXT:    andpd %xmm5, %xmm1
+; SSE41-NEXT:    roundpd $10, %xmm0, %xmm0
+; SSE41-NEXT:    andpd %xmm4, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_maskz_512_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %ymm3, %ymm1, %ymm3
+; AVX-NEXT:    vcmpeqpd %ymm2, %ymm0, %ymm2
+; AVX-NEXT:    vroundpd $10, %ymm1, %ymm1
+; AVX-NEXT:    vandpd %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    vroundpd $10, %ymm0, %ymm0
+; AVX-NEXT:    vandpd %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_maskz_512_pd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512-NEXT:    vrndscalepd $10, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %k = fcmp oeq <8 x double> %x, %y
+  %call = call <8 x double> @llvm.ceil.v8f64(<8 x double> %x)
+  %res = select <8 x i1> %k, <8 x double> %call, <8 x double> zeroinitializer
+  ret <8 x double> %res
+}
+
+define <4 x float> @ceil_mask_ss(<4 x float> %x, <4 x float> %y, <4 x float> %w, i8 %k) nounwind {
+; SSE41-LABEL: ceil_mask_ss:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    je LBB76_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundss $10, %xmm0, %xmm2
+; SSE41-NEXT:  LBB76_2:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_ss:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    je LBB76_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB76_2:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_ss:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscaless $2, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512-NEXT:    vmovaps %xmm2, %xmm0
+; AVX512-NEXT:    retq
+  %mask = and i8 %k, 1
+  %nmask = icmp eq i8 %mask, 0
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.ceil.f32(float %s)
+  %dst = extractelement <4 x float> %w, i64 0
+  %low = select i1 %nmask, float %dst, float %call
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}
+
+define <4 x float> @ceil_maskz_ss(<4 x float> %x, <4 x float> %y, i8 %k) nounwind {
+; SSE41-LABEL: ceil_maskz_ss:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    je LBB77_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundss $10, %xmm0, %xmm2
+; SSE41-NEXT:  LBB77_2:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_maskz_ss:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    je LBB77_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB77_2:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_maskz_ss:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscaless $2, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %mask = and i8 %k, 1
+  %nmask = icmp eq i8 %mask, 0
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.ceil.f32(float %s)
+  %low = select i1 %nmask, float zeroinitializer, float %call
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}
+
+define <2 x double> @ceil_mask_sd(<2 x double> %x, <2 x double> %y, <2 x double> %w, i8 %k) nounwind {
+; SSE41-LABEL: ceil_mask_sd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    je LBB78_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundsd $10, %xmm0, %xmm2
+; SSE41-NEXT:  LBB78_2:
+; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_sd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    je LBB78_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB78_2:
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_sd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscalesd $2, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512-NEXT:    vmovapd %xmm2, %xmm0
+; AVX512-NEXT:    retq
+  %mask = and i8 %k, 1
+  %nmask = icmp eq i8 %mask, 0
+  %s = extractelement <2 x double> %x, i64 0
+  %call = tail call double @llvm.ceil.f64(double %s)
+  %dst = extractelement <2 x double> %w, i64 0
+  %low = select i1 %nmask, double %dst, double %call
+  %res = insertelement <2 x double> %y, double %low, i64 0
+  ret <2 x double> %res
+}
+
+define <2 x double> @ceil_maskz_sd(<2 x double> %x, <2 x double> %y, i8 %k) nounwind {
+; SSE41-LABEL: ceil_maskz_sd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    xorpd %xmm2, %xmm2
+; SSE41-NEXT:    je LBB79_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundsd $10, %xmm0, %xmm2
+; SSE41-NEXT:  LBB79_2:
+; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_maskz_sd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    je LBB79_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB79_2:
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_maskz_sd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscalesd $2, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %mask = and i8 %k, 1
+  %nmask = icmp eq i8 %mask, 0
+  %s = extractelement <2 x double> %x, i64 0
+  %call = tail call double @llvm.ceil.f64(double %s)
+  %low = select i1 %nmask, double zeroinitializer, double %call
+  %res = insertelement <2 x double> %y, double %low, i64 0
+  ret <2 x double> %res
+}
+
+define <4 x float> @ceil_mask_ss_trunc(<4 x float> %x, <4 x float> %y, <4 x float> %w, i16 %k) nounwind {
+; SSE41-LABEL: ceil_mask_ss_trunc:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    je LBB80_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundss $10, %xmm0, %xmm2
+; SSE41-NEXT:  LBB80_2:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_ss_trunc:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    je LBB80_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB80_2:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_ss_trunc:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscaless $2, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512-NEXT:    vmovaps %xmm2, %xmm0
+; AVX512-NEXT:    retq
+  %mask = trunc i16 %k to i1
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.ceil.f32(float %s)
+  %dst = extractelement <4 x float> %w, i64 0
+  %low = select i1 %mask, float %call, float %dst
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}
+
+define <4 x float> @ceil_maskz_ss_trunc(<4 x float> %x, <4 x float> %y, i16 %k) nounwind {
+; SSE41-LABEL: ceil_maskz_ss_trunc:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    jne LBB81_1
+; SSE41-NEXT:  ## %bb.2:
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    jmp LBB81_3
+; SSE41-NEXT:  LBB81_1:
+; SSE41-NEXT:    roundss $10, %xmm0, %xmm0
+; SSE41-NEXT:  LBB81_3:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_maskz_ss_trunc:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    jne LBB81_1
+; AVX-NEXT:  ## %bb.2:
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+; AVX-NEXT:  LBB81_1:
+; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_maskz_ss_trunc:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscaless $2, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %mask = trunc i16 %k to i1
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.ceil.f32(float %s)
+  %low = select i1 %mask, float %call, float zeroinitializer
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}
+
+define <2 x double> @ceil_mask_sd_trunc(<2 x double> %x, <2 x double> %y, <2 x double> %w, i16 %k) nounwind {
+; SSE41-LABEL: ceil_mask_sd_trunc:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    je LBB82_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundsd $10, %xmm0, %xmm2
+; SSE41-NEXT:  LBB82_2:
+; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_sd_trunc:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    je LBB82_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB82_2:
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_sd_trunc:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscalesd $2, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512-NEXT:    vmovapd %xmm2, %xmm0
+; AVX512-NEXT:    retq
+  %mask = trunc i16 %k to i1
+  %s = extractelement <2 x double> %x, i64 0
+  %call = tail call double @llvm.ceil.f64(double %s)
+  %dst = extractelement <2 x double> %w, i64 0
+  %low = select i1 %mask, double %call, double %dst
+  %res = insertelement <2 x double> %y, double %low, i64 0
+  ret <2 x double> %res
+}
+
+define <2 x double> @ceil_maskz_sd_trunc(<2 x double> %x, <2 x double> %y, i16 %k) nounwind {
+; SSE41-LABEL: ceil_maskz_sd_trunc:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    jne LBB83_1
+; SSE41-NEXT:  ## %bb.2:
+; SSE41-NEXT:    xorpd %xmm0, %xmm0
+; SSE41-NEXT:    jmp LBB83_3
+; SSE41-NEXT:  LBB83_1:
+; SSE41-NEXT:    roundsd $10, %xmm0, %xmm0
+; SSE41-NEXT:  LBB83_3:
+; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_maskz_sd_trunc:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    jne LBB83_1
+; AVX-NEXT:  ## %bb.2:
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX-NEXT:    retq
+; AVX-NEXT:  LBB83_1:
+; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_maskz_sd_trunc:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscalesd $2, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %mask = trunc i16 %k to i1
+  %s = extractelement <2 x double> %x, i64 0
+  %call = tail call double @llvm.ceil.f64(double %s)
+  %low = select i1 %mask, double %call, double zeroinitializer
+  %res = insertelement <2 x double> %y, double %low, i64 0
+  ret <2 x double> %res
+}
+
+define <4 x float> @ceil_mask_ss_mask8(<4 x float> %x, <4 x float> %y, <4 x float> %w) nounwind {
+; SSE41-LABEL: ceil_mask_ss_mask8:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    movaps %xmm0, %xmm3
+; SSE41-NEXT:    cmpeqps %xmm1, %xmm3
+; SSE41-NEXT:    pextrb $0, %xmm3, %eax
+; SSE41-NEXT:    testb $1, %al
+; SSE41-NEXT:    je LBB84_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundss $10, %xmm0, %xmm2
+; SSE41-NEXT:  LBB84_2:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_ss_mask8:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm3
+; AVX-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    je LBB84_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB84_2:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: ceil_mask_ss_mask8:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vrndscaless $2, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512F-NEXT:    vmovaps %xmm2, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: ceil_mask_ss_mask8:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqps %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscaless $2, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512VL-NEXT:    vmovaps %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
+  %mask1 = fcmp oeq <4 x float> %x, %y
+  %mask = extractelement <4 x i1> %mask1, i64 0
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.ceil.f32(float %s)
+  %dst = extractelement <4 x float> %w, i64 0
+  %low = select i1 %mask, float %call, float %dst
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}
+
+define <4 x float> @ceil_maskz_ss_mask8(<4 x float> %x, <4 x float> %y) nounwind {
+; SSE41-LABEL: ceil_maskz_ss_mask8:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    movaps %xmm0, %xmm2
+; SSE41-NEXT:    cmpeqps %xmm1, %xmm2
+; SSE41-NEXT:    pextrb $0, %xmm2, %eax
+; SSE41-NEXT:    testb $1, %al
+; SSE41-NEXT:    jne LBB85_1
+; SSE41-NEXT:  ## %bb.2:
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    jmp LBB85_3
+; SSE41-NEXT:  LBB85_1:
+; SSE41-NEXT:    roundss $10, %xmm0, %xmm0
+; SSE41-NEXT:  LBB85_3:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_maskz_ss_mask8:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    jne LBB85_1
+; AVX-NEXT:  ## %bb.2:
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+; AVX-NEXT:  LBB85_1:
+; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: ceil_maskz_ss_mask8:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vrndscaless $2, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: ceil_maskz_ss_mask8:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqps %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscaless $2, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    retq
+  %mask1 = fcmp oeq <4 x float> %x, %y
+  %mask = extractelement <4 x i1> %mask1, i64 0
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.ceil.f32(float %s)
+  %low = select i1 %mask, float %call, float zeroinitializer
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}
+
+define <2 x double> @ceil_mask_sd_mask8(<2 x double> %x, <2 x double> %y, <2 x double> %w) nounwind {
+; SSE41-LABEL: ceil_mask_sd_mask8:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    movapd %xmm0, %xmm3
+; SSE41-NEXT:    cmpeqpd %xmm1, %xmm3
+; SSE41-NEXT:    pextrb $0, %xmm3, %eax
+; SSE41-NEXT:    testb $1, %al
+; SSE41-NEXT:    je LBB86_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundsd $10, %xmm0, %xmm2
+; SSE41-NEXT:  LBB86_2:
+; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_sd_mask8:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm3
+; AVX-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    je LBB86_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB86_2:
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: ceil_mask_sd_mask8:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vrndscalesd $2, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512F-NEXT:    vmovapd %xmm2, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: ceil_mask_sd_mask8:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqpd %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscalesd $2, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512VL-NEXT:    vmovapd %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
+  %mask1 = fcmp oeq <2 x double> %x, %y
+  %mask = extractelement <2 x i1> %mask1, i64 0
+  %s = extractelement <2 x double> %x, i64 0
+  %call = tail call double @llvm.ceil.f64(double %s)
+  %dst = extractelement <2 x double> %w, i64 0
+  %low = select i1 %mask, double %call, double %dst
+  %res = insertelement <2 x double> %y, double %low, i64 0
+  ret <2 x double> %res
+}
+
+define <2 x double> @ceil_maskz_sd_mask8(<2 x double> %x, <2 x double> %y) nounwind {
+; SSE41-LABEL: ceil_maskz_sd_mask8:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    movapd %xmm0, %xmm2
+; SSE41-NEXT:    cmpeqpd %xmm1, %xmm2
+; SSE41-NEXT:    pextrb $0, %xmm2, %eax
+; SSE41-NEXT:    testb $1, %al
+; SSE41-NEXT:    jne LBB87_1
+; SSE41-NEXT:  ## %bb.2:
+; SSE41-NEXT:    xorpd %xmm0, %xmm0
+; SSE41-NEXT:    jmp LBB87_3
+; SSE41-NEXT:  LBB87_1:
+; SSE41-NEXT:    roundsd $10, %xmm0, %xmm0
+; SSE41-NEXT:  LBB87_3:
+; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_maskz_sd_mask8:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vpextrb $0, %xmm2, %eax
+; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    jne LBB87_1
+; AVX-NEXT:  ## %bb.2:
+; AVX-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX-NEXT:    retq
+; AVX-NEXT:  LBB87_1:
+; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; AVX-NEXT:    retq
+;
+; AVX512F-LABEL: ceil_maskz_sd_mask8:
+; AVX512F:       ## %bb.0:
+; AVX512F-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vrndscalesd $2, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: ceil_maskz_sd_mask8:
+; AVX512VL:       ## %bb.0:
+; AVX512VL-NEXT:    vcmpeqpd %xmm1, %xmm0, %k1
+; AVX512VL-NEXT:    vrndscalesd $2, %xmm0, %xmm1, %xmm0 {%k1} {z}
+; AVX512VL-NEXT:    retq
+  %mask1 = fcmp oeq <2 x double> %x, %y
+  %mask = extractelement <2 x i1> %mask1, i64 0
+  %s = extractelement <2 x double> %x, i64 0
+  %call = tail call double @llvm.ceil.f64(double %s)
+  %low = select i1 %mask, double %call, double zeroinitializer
+  %res = insertelement <2 x double> %y, double %low, i64 0
+  ret <2 x double> %res
+}