[X86] VRNDSCALE* folding from masked and scalar ffloor and fceil patterns

[mike.dvoretsky]

Summary

This patch handles back-end folding of generic patterns created by D48067 in cases where the instruction isn't a straightforward packed values rounding operation, but a masked operation or a scalar operation.

[craig.topper]

Tests?

[mike.dvoretsky]

Added tests for folding. Also made changes to enable the new version of D45202.

[craig.topper]

llvm/lib/Target/X86/X86ISelLowering.cpp

30539

Can we just do this with isel patterns like we do for ADDSS?

llvm/test/CodeGen/X86/vec_floor.ll

873

Can you generate %k from a compare instruction rather than passing in a X x i1 type. It will make the code a little cleaner since we won't have to extend and split the mask in such crazy ways.

[mike.dvoretsky]

llvm/lib/Target/X86/X86ISelLowering.cpp

30539

I've considered that, but decided to fold it here. To do it in .td patterns we'd need to add 4 new patterns in 2 separate files. 32 and 64 bit patterns would need to be added for VROUNDS* on AVX and ROUNDS* on SSE4.1. Writing this pattern here both makes it easier to track and produces less check complexity.

[craig.topper]

llvm/lib/Target/X86/X86ISelLowering.cpp

30533

There's a signed vs unsigned comparison warning on this line.

[mike.dvoretsky]

Changed the folding to use isel patterns from D47012.

[craig.topper]

llvm/lib/Target/X86/X86InstrAVX512.td

8606

Do we have test cases covering this pattern? I can't find any zero extend instructions

[mike.dvoretsky]

Added zero extension of mask to i32 in the masked scalar tests and added more ways to represent the mask, testing the 8-bit mask pattern among others. 16-bit mask patterns removed due to scalar_to_vector errors.

[craig.topper]

llvm/lib/Target/X86/X86InstrAVX512.td

8605

Why HasVLX? Shouldn't scalar instructions be valid under HasAVX512?

9777

What about zero masking?

[mike.dvoretsky]

Corrected the scalar pattern predicates, added packed zero-masked instruction patterns and tests to cover zero-masking. Changed the RUN line of vec_floor.ll to give different results for AVX512F and AVX512VL where needed (e.g. in 128- and 256-bit masked operations).

[mike.dvoretsky]

Added tests for floor intrinsics and masked scalar double patterns to cover all introduced isel patterns.

[craig.topper]

LGTM

[mike.dvoretsky]

Updated to support D48067 instead of D45202. Removed IntrinsicsX86.td changes since the lowering is now in InstCombine, changed the masked scalar pattern tests to exclude mask extension that was in the previous versions.

[mike.dvoretsky]

Updated the AVX512 isel pattern to account for the instruction record name change.

[craig.topper]

LGTM

llvm/include/llvm/IR/IntrinsicsX86.td

def int_x86_ssse3_pabs_d     : GCCBuiltin<"__builtin_ia32_pabsd">,
               Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>;
 }
 
 //===----------------------------------------------------------------------===//
 // SSE4.1
 
 // FP rounding ops
 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
-  def int_x86_sse41_round_ss        : GCCBuiltin<"__builtin_ia32_roundss">,
+  def int_x86_sse41_round_ss        :
               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
                          llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_sse41_round_ps        : GCCBuiltin<"__builtin_ia32_roundps">,
+  def int_x86_sse41_round_ps        :
               Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty,
                          llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_sse41_round_sd        : GCCBuiltin<"__builtin_ia32_roundsd">,
+  def int_x86_sse41_round_sd        :
               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
                          llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_sse41_round_pd        : GCCBuiltin<"__builtin_ia32_roundpd">,
+  def int_x86_sse41_round_pd        :
               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,
                          llvm_i32_ty], [IntrNoMem]>;
 }
 
 // Vector min element
 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
   def int_x86_sse41_phminposuw     : GCCBuiltin<"__builtin_ia32_phminposuw128">,
               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty],
def int_x86_avx_sqrt_ps_256 : GCCBuiltin<"__builtin_ia32_sqrtps256">,
         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty], [IntrNoMem]>;
 
   def int_x86_avx_rsqrt_ps_256 : GCCBuiltin<"__builtin_ia32_rsqrtps256">,
         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty], [IntrNoMem]>;
 
   def int_x86_avx_rcp_ps_256 : GCCBuiltin<"__builtin_ia32_rcpps256">,
         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty], [IntrNoMem]>;
 
-  def int_x86_avx_round_pd_256 : GCCBuiltin<"__builtin_ia32_roundpd256">,
+  def int_x86_avx_round_pd_256 :
         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty,
                   llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx_round_ps_256 : GCCBuiltin<"__builtin_ia32_roundps256">,
+  def int_x86_avx_round_ps_256 :
         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty,
                   llvm_i32_ty], [IntrNoMem]>;
 }
 
 // Horizontal ops
 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
   def int_x86_avx_hadd_pd_256 : GCCBuiltin<"__builtin_ia32_haddpd256">,
         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty,
def int_x86_avx512_mask_cvtuqq2ps_256 :
           [IntrNoMem]>;
 
   def int_x86_avx512_mask_cvtuqq2ps_512 :
         GCCBuiltin<"__builtin_ia32_cvtuqq2ps512_mask">,
           Intrinsic<[llvm_v8f32_ty],
           [llvm_v8i64_ty, llvm_v8f32_ty,  llvm_i8_ty,  llvm_i32_ty],
           [IntrNoMem]>;
 
-  def int_x86_avx512_mask_rndscale_pd_128 : GCCBuiltin<"__builtin_ia32_rndscalepd_128_mask">,
+  def int_x86_avx512_mask_rndscale_pd_128 :
         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty,
                                      llvm_v2f64_ty,  llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_rndscale_pd_256 : GCCBuiltin<"__builtin_ia32_rndscalepd_256_mask">,
+  def int_x86_avx512_mask_rndscale_pd_256 :
         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty,
                                      llvm_v4f64_ty,  llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_rndscale_pd_512 : GCCBuiltin<"__builtin_ia32_rndscalepd_mask">,
+  def int_x86_avx512_mask_rndscale_pd_512 :
         Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty,
                                      llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_rndscale_ps_128 : GCCBuiltin<"__builtin_ia32_rndscaleps_128_mask">,
+  def int_x86_avx512_mask_rndscale_ps_128 :
         Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty,
                                      llvm_v4f32_ty,  llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_rndscale_ps_256 : GCCBuiltin<"__builtin_ia32_rndscaleps_256_mask">,
+  def int_x86_avx512_mask_rndscale_ps_256 :
         Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty,
                                      llvm_v8f32_ty,  llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_rndscale_ps_512 : GCCBuiltin<"__builtin_ia32_rndscaleps_mask">,
+  def int_x86_avx512_mask_rndscale_ps_512 :
         Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty,
                                      llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_reduce_pd_128 : GCCBuiltin<"__builtin_ia32_reducepd128_mask">,
         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty,
                                      llvm_v2f64_ty,  llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_reduce_pd_256 : GCCBuiltin<"__builtin_ia32_reducepd256_mask">,
         Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty,
                                      llvm_v4f64_ty,  llvm_i8_ty], [IntrNoMem]>;
def int_x86_avx512_mask_sub_sd_round : GCCBuiltin<"__builtin_ia32_subsd_round_mask">,
                      llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_max_sd_round : GCCBuiltin<"__builtin_ia32_maxsd_round_mask">,
           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
                      llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
   def int_x86_avx512_mask_min_sd_round : GCCBuiltin<"__builtin_ia32_minsd_round_mask">,
           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
                      llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>;
 
-  def int_x86_avx512_mask_rndscale_ss : GCCBuiltin<"__builtin_ia32_rndscaless_round_mask">,
+  def int_x86_avx512_mask_rndscale_ss :
           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
                                      llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
                                      [IntrNoMem]>;
-  def int_x86_avx512_mask_rndscale_sd : GCCBuiltin<"__builtin_ia32_rndscalesd_round_mask">,
+  def int_x86_avx512_mask_rndscale_sd :
           Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty,
                                       llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
                                      [IntrNoMem]>;
   def int_x86_avx512_mask_range_ss : GCCBuiltin<"__builtin_ia32_rangess128_round_mask">,
           Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty,
                                      llvm_i8_ty, llvm_i32_ty, llvm_i32_ty],
                                      [IntrNoMem]>;
   def int_x86_avx512_mask_range_sd : GCCBuiltin<"__builtin_ia32_rangesd128_round_mask">,

llvm/lib/Target/X86/X86ISelLowering.cpp

static SDValue combineShuffleOfConcatUndef(SDNode *N, SelectionDAG &DAG,
   if (VT.getVectorElementType() != MVT::i32 &&
       VT.getVectorElementType() != MVT::i64 &&
       VT.getVectorElementType() != MVT::f32 &&
       VT.getVectorElementType() != MVT::f64)
     return SDValue();
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
 
   // Check that both sources are concats with undef.
   if (N0.getOpcode() != ISD::CONCAT_VECTORS ||
       N1.getOpcode() != ISD::CONCAT_VECTORS || N0.getNumOperands() != 2 ||
       N1.getNumOperands() != 2 || !N0.getOperand(1).isUndef() ||
       !N1.getOperand(1).isUndef())
     return SDValue();
 
   // Construct the new shuffle mask. Elements from the first source retain their
   // index, but elements from the second source no longer need to skip an undef.
   SmallVector<int, 8> Mask;
   int NumElts = VT.getVectorNumElements();
 
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
   for (int Elt : SVOp->getMask())
static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG) {
   SDValue Src = N->getOperand(0);
 
   // If this is a scalar to vector to v1i1 from an AND with 1, bypass the and.
   // This occurs frequently in our masked scalar intrinsic code and our
   // floating point select lowering with AVX512.
   // TODO: SimplifyDemandedBits instead?
   if (VT == MVT::v1i1 && Src.getOpcode() == ISD::AND && Src.hasOneUse())
     if (auto *C = dyn_cast<ConstantSDNode>(Src.getOperand(1)))
-      if (C->getAPIntValue().isOneValue())
-        return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), MVT::v1i1,
-                           Src.getOperand(0));
+      if (C->getAPIntValue().isOneValue()) {
+        SDValue Mask = Src.getOperand(0);
+        if (Mask.getOpcode() == ISD::TRUNCATE &&
+            Mask.getOperand(0).getValueType() != MVT::i16)
+          Mask = Mask.getOperand(0);
+        return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), MVT::v1i1, Mask);
+      }
+
+  // The result of AND may also be truncated. This occurs in code for lowered
+  // masked scalar intrinsics.
+  if (VT == MVT::v1i1 && Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse() &&
+      Src.getOperand(0).getOpcode() == ISD::AND &&
+      Src.getOperand(0).hasOneUse())
+    if (auto *C = dyn_cast<ConstantSDNode>(Src.getOperand(0).getOperand(1)))
+      if (C->getAPIntValue().isOneValue()) {
+        SDValue Mask = Src.getOperand(0).getOperand(0);
+        if (Mask.getOpcode() == ISD::TRUNCATE &&
+            Mask.getOperand(0).getValueType() != MVT::i16)
+          Mask = Mask.getOperand(0);
+        // Check if the initial value is an i16. scalar_to_vector fails to
+        // select for that type, so the combine should be aborted.
+        if (Mask.getValueType() == MVT::i16)
+          return SDValue();
+        return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), MVT::v1i1, Mask);
+      }
 
   return SDValue();
 }
 
 // Simplify PMULDQ and PMULUDQ operations.
 static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
                              TargetLowering::DAGCombinerInfo &DCI) {
   SDValue LHS = N->getOperand(0);

llvm/lib/Target/X86/X86InstrAVX512.td

let Predicates = [BasePredicate] in {
 
     def : Pat<(Move _.VT:$src1, (scalar_to_vector (X86selects Mask,
                (OpNode (extractelt _.VT:$src2, (iPTR 0))), ZeroFP))),
               (!cast<Instruction>("V"#OpcPrefix#r_Intkz)
                OutMask, _.VT:$src1, _.VT:$src2, (i32 ImmV))>;
   }
 }
 
+defm : avx512_masked_scalar_imm<ffloor, "RNDSCALESS", X86Movss,
+                                (v1i1 (scalar_to_vector GR32:$mask)),
+                                v4f32x_info, fp32imm0, 0x01,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasVLX>;
+defm : avx512_masked_scalar_imm<ffloor, "RNDSCALESS", X86Movss,
+                                (v1i1 (scalar_to_vector GR8:$mask)),
+                                v4f32x_info, fp32imm0, 0x01,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasVLX>;
+defm : avx512_masked_scalar_imm<fceil, "RNDSCALESS", X86Movss,
+                                (v1i1 (scalar_to_vector GR32:$mask)),
+                                v4f32x_info, fp32imm0, 0x02,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasVLX>;
+defm : avx512_masked_scalar_imm<fceil, "RNDSCALESS", X86Movss,
+                                (v1i1 (scalar_to_vector GR8:$mask)),
+                                v4f32x_info, fp32imm0, 0x02,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasVLX>;
+defm : avx512_masked_scalar_imm<ffloor, "RNDSCALESD", X86Movsd,
+                                (v1i1 (scalar_to_vector GR32:$mask)),
+                                v2f64x_info, fp64imm0, 0x01,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasVLX>;
+defm : avx512_masked_scalar_imm<ffloor, "RNDSCALESD", X86Movsd,
+                                (v1i1 (scalar_to_vector GR8:$mask)),
+                                v2f64x_info, fp64imm0, 0x01,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasVLX>;
+defm : avx512_masked_scalar_imm<fceil, "RNDSCALESD", X86Movsd,
+                                (v1i1 (scalar_to_vector GR32:$mask)),
+                                v2f64x_info, fp64imm0, 0x02,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasVLX>;
+defm : avx512_masked_scalar_imm<fceil, "RNDSCALESD", X86Movsd,
+                                (v1i1 (scalar_to_vector GR8:$mask)),
+                                v2f64x_info, fp64imm0, 0x02,
+                                (COPY_TO_REGCLASS $mask, VK1WM), HasVLX>;
+
+
 //-------------------------------------------------
 // Integer truncate and extend operations
 //-------------------------------------------------
 
 multiclass avx512_trunc_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
                               X86FoldableSchedWrite sched, X86VectorVTInfo SrcInfo,
                               X86VectorVTInfo DestInfo, X86MemOperand x86memop> {
   let ExeDomain = DestInfo.ExeDomain in
defm VGETMANTSD: avx512_common_fp_sae_scalar_imm<"vgetmantsd", f64x_info,
       AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
 defm VGETMANTSS: avx512_common_fp_sae_scalar_imm<"vgetmantss", f32x_info,
       0x27, X86GetMants, X86GetMantsRnd, SchedWriteFRnd, HasAVX512>,
       AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
 
 let Predicates = [HasAVX512] in {
 def : Pat<(v16f32 (ffloor VR512:$src)),
           (VRNDSCALEPSZrri VR512:$src, (i32 0x9))>;
+def : Pat<(v16f32 (vselect VK16WM:$mask, (ffloor VR512:$src), VR512:$dst)),
+          (VRNDSCALEPSZrrik VR512:$dst, VK16WM:$mask, VR512:$src, (i32 0x9))>;
 def : Pat<(v16f32 (fnearbyint VR512:$src)),
           (VRNDSCALEPSZrri VR512:$src, (i32 0xC))>;
 def : Pat<(v16f32 (fceil VR512:$src)),
           (VRNDSCALEPSZrri VR512:$src, (i32 0xA))>;
+def : Pat<(v16f32 (vselect VK16WM:$mask, (fceil VR512:$src), VR512:$dst)),
+          (VRNDSCALEPSZrrik VR512:$dst, VK16WM:$mask, VR512:$src, (i32 0xA))>;
 def : Pat<(v16f32 (frint VR512:$src)),
           (VRNDSCALEPSZrri VR512:$src, (i32 0x4))>;
 def : Pat<(v16f32 (ftrunc VR512:$src)),
           (VRNDSCALEPSZrri VR512:$src, (i32 0xB))>;
 
 def : Pat<(v8f64 (ffloor VR512:$src)),
           (VRNDSCALEPDZrri VR512:$src, (i32 0x9))>;
+def : Pat<(v8f64 (vselect VK8WM:$mask, (ffloor VR512:$src), VR512:$dst)),
+          (VRNDSCALEPDZrrik VR512:$dst, VK8WM:$mask, VR512:$src, (i32 0x9))>;
 def : Pat<(v8f64 (fnearbyint VR512:$src)),
           (VRNDSCALEPDZrri VR512:$src, (i32 0xC))>;
 def : Pat<(v8f64 (fceil VR512:$src)),
           (VRNDSCALEPDZrri VR512:$src, (i32 0xA))>;
+def : Pat<(v8f64 (vselect VK8WM:$mask, (fceil VR512:$src), VR512:$dst)),
+          (VRNDSCALEPDZrrik VR512:$dst, VK8WM:$mask, VR512:$src, (i32 0xA))>;
 def : Pat<(v8f64 (frint VR512:$src)),
           (VRNDSCALEPDZrri VR512:$src, (i32 0x4))>;
 def : Pat<(v8f64 (ftrunc VR512:$src)),
           (VRNDSCALEPDZrri VR512:$src, (i32 0xB))>;
 }
 
 let Predicates = [HasVLX] in {
 def : Pat<(v4f32 (ffloor VR128X:$src)),
           (VRNDSCALEPSZ128rri VR128X:$src, (i32 0x9))>;
+def : Pat<(v4f32 (vselect VK4WM:$mask, (ffloor VR128X:$src), VR128X:$dst)),
+          (VRNDSCALEPSZ128rrik VR128X:$dst, VK4WM:$mask, VR128X:$src, (i32 0x9))>;
 def : Pat<(v4f32 (fnearbyint VR128X:$src)),
           (VRNDSCALEPSZ128rri VR128X:$src, (i32 0xC))>;
 def : Pat<(v4f32 (fceil VR128X:$src)),
           (VRNDSCALEPSZ128rri VR128X:$src, (i32 0xA))>;
+def : Pat<(v4f32 (vselect VK4WM:$mask, (fceil VR128X:$src), VR128X:$dst)),
+          (VRNDSCALEPSZ128rrik VR128X:$dst, VK4WM:$mask, VR128X:$src, (i32 0xA))>;
 def : Pat<(v4f32 (frint VR128X:$src)),
           (VRNDSCALEPSZ128rri VR128X:$src, (i32 0x4))>;
 def : Pat<(v4f32 (ftrunc VR128X:$src)),
           (VRNDSCALEPSZ128rri VR128X:$src, (i32 0xB))>;
 
 def : Pat<(v2f64 (ffloor VR128X:$src)),
           (VRNDSCALEPDZ128rri VR128X:$src, (i32 0x9))>;
+def : Pat<(v2f64 (vselect VK2WM:$mask, (ffloor VR128X:$src), VR128X:$dst)),
+          (VRNDSCALEPDZ128rrik VR128X:$dst, VK2WM:$mask, VR128X:$src, (i32 0x9))>;
 def : Pat<(v2f64 (fnearbyint VR128X:$src)),
           (VRNDSCALEPDZ128rri VR128X:$src, (i32 0xC))>;
 def : Pat<(v2f64 (fceil VR128X:$src)),
           (VRNDSCALEPDZ128rri VR128X:$src, (i32 0xA))>;
+def : Pat<(v2f64 (vselect VK2WM:$mask, (fceil VR128X:$src), VR128X:$dst)),
+          (VRNDSCALEPDZ128rrik VR128X:$dst, VK2WM:$mask, VR128X:$src, (i32 0xA))>;
 def : Pat<(v2f64 (frint VR128X:$src)),
           (VRNDSCALEPDZ128rri VR128X:$src, (i32 0x4))>;
 def : Pat<(v2f64 (ftrunc VR128X:$src)),
           (VRNDSCALEPDZ128rri VR128X:$src, (i32 0xB))>;
 
 def : Pat<(v8f32 (ffloor VR256X:$src)),
           (VRNDSCALEPSZ256rri VR256X:$src, (i32 0x9))>;
+def : Pat<(v8f32 (vselect VK8WM:$mask, (ffloor VR256X:$src), VR256X:$dst)),
+          (VRNDSCALEPSZ256rrik VR256X:$dst, VK8WM:$mask, VR256X:$src, (i32 0x9))>;
 def : Pat<(v8f32 (fnearbyint VR256X:$src)),
           (VRNDSCALEPSZ256rri VR256X:$src, (i32 0xC))>;
 def : Pat<(v8f32 (fceil VR256X:$src)),
           (VRNDSCALEPSZ256rri VR256X:$src, (i32 0xA))>;
+def : Pat<(v8f32 (vselect VK8WM:$mask, (fceil VR256X:$src), VR256X:$dst)),
+          (VRNDSCALEPSZ256rrik VR256X:$dst, VK8WM:$mask, VR256X:$src, (i32 0xA))>;
 def : Pat<(v8f32 (frint VR256X:$src)),
           (VRNDSCALEPSZ256rri VR256X:$src, (i32 0x4))>;
 def : Pat<(v8f32 (ftrunc VR256X:$src)),
           (VRNDSCALEPSZ256rri VR256X:$src, (i32 0xB))>;
 
 def : Pat<(v4f64 (ffloor VR256X:$src)),
           (VRNDSCALEPDZ256rri VR256X:$src, (i32 0x9))>;
+def : Pat<(v4f64 (vselect VK4WM:$mask, (ffloor VR256X:$src), VR256X:$dst)),
+          (VRNDSCALEPDZ256rrik VR256X:$dst, VK4WM:$mask, VR256X:$src, (i32 0x9))>;
 def : Pat<(v4f64 (fnearbyint VR256X:$src)),
           (VRNDSCALEPDZ256rri VR256X:$src, (i32 0xC))>;
 def : Pat<(v4f64 (fceil VR256X:$src)),
           (VRNDSCALEPDZ256rri VR256X:$src, (i32 0xA))>;
+def : Pat<(v4f64 (vselect VK4WM:$mask, (fceil VR256X:$src), VR256X:$dst)),
+          (VRNDSCALEPDZ256rrik VR256X:$dst, VK4WM:$mask, VR256X:$src, (i32 0xA))>;
 def : Pat<(v4f64 (frint VR256X:$src)),
           (VRNDSCALEPDZ256rri VR256X:$src, (i32 0x4))>;
 def : Pat<(v4f64 (ftrunc VR256X:$src)),
           (VRNDSCALEPDZ256rri VR256X:$src, (i32 0xB))>;
 }
 
 multiclass avx512_shuff_packed_128_common<bits<8> opc, string OpcodeStr,
                                           X86FoldableSchedWrite sched, X86VectorVTInfo _,

llvm/lib/Target/X86/X86InstrSSE.td

let Predicates = [UseSSE41] in {
   def : Pat<(v2f64 (fceil VR128:$src)),
             (ROUNDPDr VR128:$src, (i32 0xA))>;
   def : Pat<(v2f64 (frint VR128:$src)),
             (ROUNDPDr VR128:$src, (i32 0x4))>;
   def : Pat<(v2f64 (ftrunc VR128:$src)),
             (ROUNDPDr VR128:$src, (i32 0xB))>;
 }
 
+defm : scalar_unary_math_imm_patterns<ffloor, "ROUNDSS", X86Movss,
+                                      v4f32, 0x01, UseSSE41>;
+defm : scalar_unary_math_imm_patterns<fceil, "ROUNDSS", X86Movss,
+                                      v4f32, 0x02, UseSSE41>;
+defm : scalar_unary_math_imm_patterns<ffloor, "ROUNDSD", X86Movsd,
+                                      v2f64, 0x01, UseSSE41>;
+defm : scalar_unary_math_imm_patterns<fceil, "ROUNDSD", X86Movsd,
+                                      v2f64, 0x02, UseSSE41>;
+
 //===----------------------------------------------------------------------===//
 // SSE4.1 - Packed Bit Test
 //===----------------------------------------------------------------------===//
 
 // ptest instruction we'll lower to this in X86ISelLowering primarily from
 // the intel intrinsic that corresponds to this.
 let Defs = [EFLAGS], Predicates = [HasAVX] in {
 def VPTESTrr  : SS48I<0x17, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src2),

llvm/test/CodeGen/X86/vec_floor.ll

 ;
 ; AVX512-LABEL: const_trunc_v4f32:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vmovaps {{.*#+}} xmm0 = [-3.000000e+00,6.000000e+00,-9.000000e+00,2.000000e+00]
 ; AVX512-NEXT:    retq
   %t = call <4 x float> @llvm.trunc.v4f32(<4 x float> <float -3.5, float 6.0, float -9.0, float 2.5>)
   ret <4 x float> %t
 }
+
+define <4 x float> @ceil_ss(<4 x float> %x, <4 x float> %y) nounwind {
+; SSE41-LABEL: ceil_ss:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundss $2, %xmm0, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_ss:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vroundss $2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_ss:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vroundss $2, %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %s = extractelement <4 x float> %x, i32 0
+  %call = call float @llvm.ceil.f32(float %s)
+  %res = insertelement <4 x float> %y, float %call, i32 0
+  ret <4 x float> %res
+}
+declare float @llvm.ceil.f32(float %s)
+
+define <2 x double> @ceil_sd(<2 x double> %x, <2 x double> %y) nounwind {
+; SSE41-LABEL: ceil_sd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundsd $2, %xmm0, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_sd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vroundsd $2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_sd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vroundsd $2, %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %s = extractelement <2 x double> %x, i32 0
+  %call = call double @llvm.ceil.f64(double %s)
+  %res = insertelement <2 x double> %y, double %call, i32 0
+  ret <2 x double> %res
+}
+declare double @llvm.ceil.f64(double %s)
+
+define <4 x float> @ceil_mask_128_ps(<4 x float> %x, <4 x float> %y) nounwind {
+; SSE41-LABEL: ceil_mask_128_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundps $10, %xmm0, %xmm2
+; SSE41-NEXT:    cmpeqps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_128_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vroundps $10, %xmm0, %xmm0
+; AVX-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_128_ps:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqps %xmm1, %xmm0, %k1
+; AVX512-NEXT:    vrndscaleps $10, %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovaps %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %k = fcmp oeq <4 x float> %x, %y
+  %call = call <4 x float> @llvm.ceil.v4f32(<4 x float> %x)
+  %res = select <4 x i1> %k, <4 x float> %call, <4 x float> %y
+  ret <4 x float> %res
+}
+
+define <2 x double> @ceil_mask_128_pd(<2 x double> %x, <2 x double> %y) nounwind {
+; SSE41-LABEL: ceil_mask_128_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundpd $10, %xmm0, %xmm2
+; SSE41-NEXT:    cmpeqpd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_128_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vroundpd $10, %xmm0, %xmm0
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_128_pd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqpd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    vrndscalepd $10, %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovapd %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %k = fcmp oeq <2 x double> %x, %y
+  %call = call <2 x double> @llvm.ceil.v2f64(<2 x double> %x)
+  %res = select <2 x i1> %k, <2 x double> %call, <2 x double> %y
+  ret <2 x double> %res
+}
+
+define <8 x float> @ceil_mask_256_ps(<8 x float> %x, <8 x float> %y) nounwind {
+; SSE41-LABEL: ceil_mask_256_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundps $10, %xmm1, %xmm4
+; SSE41-NEXT:    cmpeqps %xmm3, %xmm1
+; SSE41-NEXT:    roundps $10, %xmm0, %xmm5
+; SSE41-NEXT:    cmpeqps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    movaps %xmm3, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_256_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %ymm1, %ymm0, %ymm2
+; AVX-NEXT:    vroundps $10, %ymm0, %ymm0
+; AVX-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_256_ps:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqps %ymm1, %ymm0, %k1
+; AVX512-NEXT:    vrndscaleps $10, %ymm0, %ymm1 {%k1}
+; AVX512-NEXT:    vmovaps %ymm1, %ymm0
+; AVX512-NEXT:    retq
+  %k = fcmp oeq <8 x float> %x, %y
+  %call = call <8 x float> @llvm.ceil.v8f32(<8 x float> %x)
+  %res = select <8 x i1> %k, <8 x float> %call, <8 x float> %y
+  ret <8 x float> %res
+}
+
+define <4 x double> @ceil_mask_256_pd(<4 x double> %x, <4 x double> %y) nounwind {
+; SSE41-LABEL: ceil_mask_256_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundpd $10, %xmm1, %xmm4
+; SSE41-NEXT:    cmpeqpd %xmm3, %xmm1
+; SSE41-NEXT:    roundpd $10, %xmm0, %xmm5
+; SSE41-NEXT:    cmpeqpd %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_256_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm2
+; AVX-NEXT:    vroundpd $10, %ymm0, %ymm0
+; AVX-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_256_pd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqpd %ymm1, %ymm0, %k1
+; AVX512-NEXT:    vrndscalepd $10, %ymm0, %ymm1 {%k1}
+; AVX512-NEXT:    vmovapd %ymm1, %ymm0
+; AVX512-NEXT:    retq
+  %k = fcmp oeq <4 x double> %x, %y
+  %call = call <4 x double> @llvm.ceil.v4f64(<4 x double> %x)
+  %res = select <4 x i1> %k, <4 x double> %call, <4 x double> %y
+  ret <4 x double> %res
+}
+
+define <16 x float> @ceil_mask_512_ps(<16 x float> %x, <16 x float> %y) nounwind {
+; SSE41-LABEL: ceil_mask_512_ps:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundps $10, %xmm3, %xmm8
+; SSE41-NEXT:    cmpeqps %xmm7, %xmm3
+; SSE41-NEXT:    roundps $10, %xmm2, %xmm9
+; SSE41-NEXT:    cmpeqps %xmm6, %xmm2
+; SSE41-NEXT:    roundps $10, %xmm1, %xmm10
+; SSE41-NEXT:    cmpeqps %xmm5, %xmm1
+; SSE41-NEXT:    roundps $10, %xmm0, %xmm11
+; SSE41-NEXT:    cmpeqps %xmm4, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm4
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm10, %xmm5
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm9, %xmm6
+; SSE41-NEXT:    movaps %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm8, %xmm7
+; SSE41-NEXT:    movaps %xmm4, %xmm0
+; SSE41-NEXT:    movaps %xmm5, %xmm1
+; SSE41-NEXT:    movaps %xmm6, %xmm2
+; SSE41-NEXT:    movaps %xmm7, %xmm3
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_512_ps:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %ymm3, %ymm1, %ymm4
+; AVX-NEXT:    vcmpeqps %ymm2, %ymm0, %ymm5
+; AVX-NEXT:    vroundps $10, %ymm1, %ymm1
+; AVX-NEXT:    vroundps $10, %ymm0, %ymm0
+; AVX-NEXT:    vblendvps %ymm5, %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    vblendvps %ymm4, %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_512_ps:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
+; AVX512-NEXT:    vrndscaleps $10, %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovaps %zmm1, %zmm0
+; AVX512-NEXT:    retq
+  %k = fcmp oeq <16 x float> %x, %y
+  %call = call <16 x float> @llvm.ceil.v16f32(<16 x float> %x)
+  %res = select <16 x i1> %k, <16 x float> %call, <16 x float> %y
+  ret <16 x float> %res
+}
+
+define <8 x double> @ceil_mask_512_pd(<8 x double> %x, <8 x double> %y) nounwind {
+; SSE41-LABEL: ceil_mask_512_pd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    roundpd $10, %xmm3, %xmm8
+; SSE41-NEXT:    cmpeqpd %xmm7, %xmm3
+; SSE41-NEXT:    roundpd $10, %xmm2, %xmm9
+; SSE41-NEXT:    cmpeqpd %xmm6, %xmm2
+; SSE41-NEXT:    roundpd $10, %xmm1, %xmm10
+; SSE41-NEXT:    cmpeqpd %xmm5, %xmm1
+; SSE41-NEXT:    roundpd $10, %xmm0, %xmm11
+; SSE41-NEXT:    cmpeqpd %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm4
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm5
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm6
+; SSE41-NEXT:    movapd %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm7
+; SSE41-NEXT:    movapd %xmm4, %xmm0
+; SSE41-NEXT:    movapd %xmm5, %xmm1
+; SSE41-NEXT:    movapd %xmm6, %xmm2
+; SSE41-NEXT:    movapd %xmm7, %xmm3
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_512_pd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqpd %ymm3, %ymm1, %ymm4
+; AVX-NEXT:    vcmpeqpd %ymm2, %ymm0, %ymm5
+; AVX-NEXT:    vroundpd $10, %ymm1, %ymm1
+; AVX-NEXT:    vroundpd $10, %ymm0, %ymm0
+; AVX-NEXT:    vblendvpd %ymm5, %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    vblendvpd %ymm4, %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_512_pd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqpd %zmm1, %zmm0, %k1
+; AVX512-NEXT:    vrndscalepd $10, %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovapd %zmm1, %zmm0
+; AVX512-NEXT:    retq
+  %k = fcmp oeq <8 x double> %x, %y
+  %call = call <8 x double> @llvm.ceil.v8f64(<8 x double> %x)
+  %res = select <8 x i1> %k, <8 x double> %call, <8 x double> %y
+  ret <8 x double> %res
+}
+
+define <4 x float> @ceil_mask_ss(<4 x float> %x, <4 x float> %y, <4 x float> %w, i16 %k) nounwind {
+; SSE41-LABEL: ceil_mask_ss:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    je LBB44_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundss $10, %xmm0, %xmm2
+; SSE41-NEXT:  LBB44_2:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_ss:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    je LBB44_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB44_2:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_ss:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscaless $2, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512-NEXT:    vmovaps %xmm2, %xmm0
+; AVX512-NEXT:    retq
+  %xk = zext i16 %k to i32
+  %mask = and i32 %xk, 1
+  %nmask = icmp eq i32 %mask, 0
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.ceil.f32(float %s)
+  %dst = extractelement <4 x float> %w, i64 0
+  %low = select i1 %nmask, float %dst, float %call
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}
+
+define <2 x double> @ceil_mask_sd(<2 x double> %x, <2 x double> %y, <2 x double> %w, i16 %k) nounwind {
+; SSE41-LABEL: ceil_mask_sd:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    je LBB45_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundsd $10, %xmm0, %xmm2
+; SSE41-NEXT:  LBB45_2:
+; SSE41-NEXT:    blendpd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_sd:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    je LBB45_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundsd $10, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB45_2:
+; AVX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm1[1]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_sd:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscalesd $2, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512-NEXT:    vmovapd %xmm2, %xmm0
+; AVX512-NEXT:    retq
+  %xk = zext i16 %k to i32
+  %mask = and i32 %xk, 1
+  %nmask = icmp eq i32 %mask, 0
+  %s = extractelement <2 x double> %x, i64 0
+  %call = tail call double @llvm.ceil.f64(double %s)
+  %dst = extractelement <2 x double> %w, i64 0
+  %low = select i1 %nmask, double %dst, double %call
+  %res = insertelement <2 x double> %y, double %low, i64 0
+  ret <2 x double> %res
+}
+
+define <4 x float> @ceil_mask_ss_trunc(<4 x float> %x, <4 x float> %y, <4 x float> %w, i16 %k) nounwind {
+; SSE41-LABEL: ceil_mask_ss_trunc:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    testb $1, %dil
+; SSE41-NEXT:    je LBB46_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundss $10, %xmm0, %xmm2
+; SSE41-NEXT:  LBB46_2:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_ss_trunc:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    testb $1, %dil
+; AVX-NEXT:    je LBB46_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB46_2:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_ss_trunc:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    kmovw %edi, %k1
+; AVX512-NEXT:    vrndscaless $2, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512-NEXT:    vmovaps %xmm2, %xmm0
+; AVX512-NEXT:    retq
+  %mask = trunc i16 %k to i1
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.ceil.f32(float %s)
+  %dst = extractelement <4 x float> %w, i64 0
+  %low = select i1 %mask, float %call, float %dst
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}
+
+define <4 x float> @ceil_mask_ss_mask8(<4 x float> %x, <4 x float> %y, <4 x float> %w) nounwind {
+; SSE41-LABEL: ceil_mask_ss_mask8:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    movaps %xmm0, %xmm3
+; SSE41-NEXT:    cmpeqps %xmm1, %xmm3
+; SSE41-NEXT:    pextrb $0, %xmm3, %eax
+; SSE41-NEXT:    testb $1, %al
+; SSE41-NEXT:    je LBB47_2
+; SSE41-NEXT:  ## %bb.1:
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    roundss $10, %xmm0, %xmm2
+; SSE41-NEXT:  LBB47_2:
+; SSE41-NEXT:    blendps {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ceil_mask_ss_mask8:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm3
+; AVX-NEXT:    vpextrb $0, %xmm3, %eax
+; AVX-NEXT:    testb $1, %al
+; AVX-NEXT:    je LBB47_2
+; AVX-NEXT:  ## %bb.1:
+; AVX-NEXT:    vroundss $10, %xmm0, %xmm0, %xmm2
+; AVX-NEXT:  LBB47_2:
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm1[1,2,3]
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: ceil_mask_ss_mask8:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vcmpeqps %xmm1, %xmm0, %k1
+; AVX512-NEXT:    vrndscaless $2, %xmm0, %xmm1, %xmm2 {%k1}
+; AVX512-NEXT:    vmovaps %xmm2, %xmm0
+; AVX512-NEXT:    retq
+  %mask1 = fcmp oeq <4 x float> %x, %y
+  %mask = extractelement <4 x i1> %mask1, i64 0
+  %s = extractelement <4 x float> %x, i64 0
+  %call = tail call float @llvm.ceil.f32(float %s)
+  %dst = extractelement <4 x float> %w, i64 0
+  %low = select i1 %mask, float %call, float %dst
+  %res = insertelement <4 x float> %y, float %low, i64 0
+  ret <4 x float> %res
+}