[FPEnv] Scalarize StrictFP vector operations

[cameron.mcinally]

Summary

Add a helper function to scalarize constrained FP operations as needed.

[craig.topper]

LGTM

llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeTypes.h

private:
   void SetScalarizedVector(SDValue Op, SDValue Result);
 
   // Vector Result Scalarization: <1 x ty> -> ty.
   void ScalarizeVectorResult(SDNode *N, unsigned ResNo);
   SDValue ScalarizeVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
   SDValue ScalarizeVecRes_BinOp(SDNode *N);
   SDValue ScalarizeVecRes_TernaryOp(SDNode *N);
   SDValue ScalarizeVecRes_UnaryOp(SDNode *N);
+  SDValue ScalarizeVecRes_StrictFPOp(SDNode *N);
   SDValue ScalarizeVecRes_InregOp(SDNode *N);
   SDValue ScalarizeVecRes_VecInregOp(SDNode *N);
 
   SDValue ScalarizeVecRes_BITCAST(SDNode *N);
   SDValue ScalarizeVecRes_BUILD_VECTOR(SDNode *N);
   SDValue ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N);
   SDValue ScalarizeVecRes_FP_ROUND(SDNode *N);
   SDValue ScalarizeVecRes_FPOWI(SDNode *N);

llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

#endif
   case ISD::SHL:
   case ISD::SRA:
   case ISD::SRL:
     R = ScalarizeVecRes_BinOp(N);
     break;
   case ISD::FMA:
     R = ScalarizeVecRes_TernaryOp(N);
     break;
+  case ISD::STRICT_FADD:
+  case ISD::STRICT_FSUB:
+  case ISD::STRICT_FMUL:
+  case ISD::STRICT_FDIV:
+  case ISD::STRICT_FSQRT:
+  case ISD::STRICT_FMA:
+  case ISD::STRICT_FPOW:
+  case ISD::STRICT_FPOWI:
+  case ISD::STRICT_FSIN:
+  case ISD::STRICT_FCOS:
+  case ISD::STRICT_FEXP:
+  case ISD::STRICT_FEXP2:
+  case ISD::STRICT_FLOG:
+  case ISD::STRICT_FLOG10:
+  case ISD::STRICT_FLOG2:
+  case ISD::STRICT_FRINT:
+  case ISD::STRICT_FNEARBYINT:
+    R = ScalarizeVecRes_StrictFPOp(N);
+    break;
   }
 
   // If R is null, the sub-method took care of registering the result.
   if (R.getNode())
     SetScalarizedVector(SDValue(N, ResNo), R);
 }
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
   SDValue LHS = GetScalarizedVector(N->getOperand(0));
   SDValue RHS = GetScalarizedVector(N->getOperand(1));
   return DAG.getNode(N->getOpcode(), SDLoc(N),
                      LHS.getValueType(), LHS, RHS, N->getFlags());
 }
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
   SDValue Op0 = GetScalarizedVector(N->getOperand(0));
   SDValue Op1 = GetScalarizedVector(N->getOperand(1));
   SDValue Op2 = GetScalarizedVector(N->getOperand(2));
   return DAG.getNode(N->getOpcode(), SDLoc(N),
                      Op0.getValueType(), Op0, Op1, Op2);
 }
 
+SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
+  EVT VT = N->getValueType(0).getVectorElementType();
+  unsigned NumOpers = N->getNumOperands();
+  SDValue Chain = N->getOperand(0);
+  EVT ValueVTs[] = {VT, MVT::Other};
+  SDLoc dl(N);
+
+  SmallVector<SDValue, 4> Opers;
+
+  // The Chain is the first operand.
+  Opers.push_back(Chain);
+
+  // Now process the remaining operands.
+  for (unsigned i = 1; i < NumOpers; ++i) {
+    SDValue Oper = N->getOperand(i);
+
+    if (Oper.getValueType().isVector())
+      Oper = GetScalarizedVector(Oper);
+
+    Opers.push_back(Oper);
+  }
+
+  SDValue Result = DAG.getNode(N->getOpcode(), dl, ValueVTs, Opers);
+
+  // Legalize the chain result - switch anything that used the old chain to
+  // use the new one.
+  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
+  return Result;
+}
+
 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
                                                        unsigned ResNo) {
   SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
   return GetScalarizedVector(Op);
 }
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
   SDValue Op = N->getOperand(0);

llvm/trunk/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -O3 -mtriple=x86_64-pc-linux < %s | FileCheck --check-prefix=COMMON --check-prefix=NO-FMA --check-prefix=FMACALL64 --check-prefix=FMACALL32 %s
 ; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+fma < %s | FileCheck -check-prefix=COMMON --check-prefix=HAS-FMA --check-prefix=FMA64 --check-prefix=FMA32 %s
 
+define <1 x float> @constrained_vector_fdiv_v1f32() {
+; NO-FMA-LABEL: constrained_vector_fdiv_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    divss {{.*}}(%rip), %xmm0
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_fdiv_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    vdivss {{.*}}(%rip), %xmm0, %xmm0
+; HAS-FMA-NEXT:    retq
+entry:
+  %div = call <1 x float> @llvm.experimental.constrained.fdiv.v1f32(
+           <1 x float> <float 1.000000e+00>,
+           <1 x float> <float 1.000000e+01>,
+           metadata !"round.dynamic",
+           metadata !"fpexcept.strict")
+  ret <1 x float> %div
+}
+
 define <2 x double> @constrained_vector_fdiv_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_fdiv_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    movapd {{.*#+}} xmm0 = [1.000000e+00,2.000000e+00]
 ; NO-FMA-NEXT:    divpd {{.*}}(%rip), %xmm0
 ; NO-FMA-NEXT:    retq
 ;
 ; HAS-FMA-LABEL: constrained_vector_fdiv_v2f64:
%div = call <4 x double> @llvm.experimental.constrained.fdiv.v4f64(
                          double 3.000000e+00, double 4.000000e+00>,
            <4 x double> <double 1.000000e+01, double 1.000000e+01,
                          double 1.000000e+01, double 1.000000e+01>,
            metadata !"round.dynamic",
            metadata !"fpexcept.strict")
   ret <4 x double> %div
 }
 
+define <1 x float> @constrained_vector_fmul_v1f32() {
+; NO-FMA-LABEL: constrained_vector_fmul_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    mulss {{.*}}(%rip), %xmm0
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_fmul_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; HAS-FMA-NEXT:    retq
+entry:
+  %mul = call <1 x float> @llvm.experimental.constrained.fmul.v1f32(
+           <1 x float> <float 0x7FF0000000000000>,
+           <1 x float> <float 2.000000e+00>,
+           metadata !"round.dynamic",
+           metadata !"fpexcept.strict")
+  ret <1 x float> %mul
+}
+
 define <2 x double> @constrained_vector_fmul_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_fmul_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    movapd {{.*#+}} xmm0 = [1.797693e+308,1.797693e+308]
 ; NO-FMA-NEXT:    mulpd {{.*}}(%rip), %xmm0
 ; NO-FMA-NEXT:    retq
 ;
 ; HAS-FMA-LABEL: constrained_vector_fmul_v2f64:
%mul = call <4 x double> @llvm.experimental.constrained.fmul.v4f64(
                          double 0x7FEFFFFFFFFFFFFF, double 0x7FEFFFFFFFFFFFFF>,
            <4 x double> <double 2.000000e+00, double 3.000000e+00,
                          double 4.000000e+00, double 5.000000e+00>,
            metadata !"round.dynamic",
            metadata !"fpexcept.strict")
   ret <4 x double> %mul
 }
 
+define <1 x float> @constrained_vector_fadd_v1f32() {
+; NO-FMA-LABEL: constrained_vector_fadd_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    addss {{.*}}(%rip), %xmm0
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_fadd_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    vaddss {{.*}}(%rip), %xmm0, %xmm0
+; HAS-FMA-NEXT:    retq
+entry:
+  %add = call <1 x float> @llvm.experimental.constrained.fadd.v1f32(
+           <1 x float> <float 0x7FF0000000000000>,
+           <1 x float> <float 1.0>,
+           metadata !"round.dynamic",
+           metadata !"fpexcept.strict")
+  ret <1 x float> %add
+}
 
 define <2 x double> @constrained_vector_fadd_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_fadd_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    movapd {{.*#+}} xmm0 = [1.797693e+308,1.797693e+308]
 ; NO-FMA-NEXT:    addpd {{.*}}(%rip), %xmm0
 ; NO-FMA-NEXT:    retq
 ;
%add = call <4 x double> @llvm.experimental.constrained.fadd.v4f64(
                          double 0x7FEFFFFFFFFFFFFF, double 0x7FEFFFFFFFFFFFFF>,
            <4 x double> <double 1.000000e+00, double 1.000000e-01,
                          double 2.000000e+00, double 2.000000e-01>,
            metadata !"round.dynamic",
            metadata !"fpexcept.strict")
   ret <4 x double> %add
 }
 
+define <1 x float> @constrained_vector_fsub_v1f32() {
+; NO-FMA-LABEL: constrained_vector_fsub_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    subss {{.*}}(%rip), %xmm0
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_fsub_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    vsubss {{.*}}(%rip), %xmm0, %xmm0
+; HAS-FMA-NEXT:    retq
+entry:
+  %sub = call <1 x float> @llvm.experimental.constrained.fsub.v1f32(
+           <1 x float> <float 0x7FF0000000000000>,
+           <1 x float> <float 1.000000e+00>,
+           metadata !"round.dynamic",
+           metadata !"fpexcept.strict")
+  ret <1 x float> %sub
+}
+
 define <2 x double> @constrained_vector_fsub_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_fsub_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    movapd {{.*#+}} xmm0 = [-1.797693e+308,-1.797693e+308]
 ; NO-FMA-NEXT:    subpd {{.*}}(%rip), %xmm0
 ; NO-FMA-NEXT:    retq
 ;
 ; HAS-FMA-LABEL: constrained_vector_fsub_v2f64:
%sub = call <4 x double> @llvm.experimental.constrained.fsub.v4f64(
                          double 0xFFEFFFFFFFFFFFFF, double 0xFFEFFFFFFFFFFFFF>,
            <4 x double> <double 1.000000e+00, double 1.000000e-01,
                          double 2.000000e+00, double 2.000000e-01>,
            metadata !"round.dynamic",
            metadata !"fpexcept.strict")
   ret <4 x double> %sub
 }
 
+define <1 x float> @constrained_vector_fma_v1f32() {
+; NO-FMA-LABEL: constrained_vector_fma_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    pushq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 16
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    callq fmaf
+; NO-FMA-NEXT:    popq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 8
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_fma_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem
+; HAS-FMA-NEXT:    retq
+entry:
+  %fma = call <1 x float> @llvm.experimental.constrained.fma.v1f32(
+           <1 x float> <float 0.5>,
+           <1 x float> <float 2.5>,
+           <1 x float> <float 4.5>,
+           metadata !"round.dynamic",
+           metadata !"fpexcept.strict")
+  ret <1 x float> %fma
+}
+
 define <2 x double> @constrained_vector_fma_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_fma_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    subq $24, %rsp
 ; NO-FMA-NEXT:    .cfi_def_cfa_offset 32
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
%fma = call <8 x float> @llvm.experimental.constrained.fma.v8f32(
                         float 11.5, float 10.5, float 9.5, float 8.5>,
            <8 x float> <float 11.5, float 10.5, float 9.5, float 8.5,
                         float 15.5, float 14.5, float 13.5, float 12.5>,
            metadata !"round.dynamic",
            metadata !"fpexcept.strict")
   ret <8 x float> %fma
 }
 
+define <1 x float> @constrained_vector_sqrt_v1f32() {
+; NO-FMA-LABEL: constrained_vector_sqrt_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    sqrtss %xmm0, %xmm0
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_sqrt_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0
+; HAS-FMA-NEXT:    retq
+entry:
+  %sqrt = call <1 x float> @llvm.experimental.constrained.sqrt.v1f32(
+                              <1 x float> <float 42.0>,
+                              metadata !"round.dynamic",
+                              metadata !"fpexcept.strict")
+  ret <1 x float> %sqrt
+}
+
 define <2 x double> @constrained_vector_sqrt_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_sqrt_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    sqrtpd {{.*}}(%rip), %xmm0
 ; NO-FMA-NEXT:    retq
 ;
 ; HAS-FMA-LABEL: constrained_vector_sqrt_v2f64:
 ; HAS-FMA:       # %bb.0: # %entry
entry:
   %sqrt = call <4 x double> @llvm.experimental.constrained.sqrt.v4f64(
                               <4 x double> <double 42.0, double 42.1,
                                             double 42.2, double 42.3>,
                               metadata !"round.dynamic",
                               metadata !"fpexcept.strict")
   ret <4 x double> %sqrt
 }
 
+define <1 x float> @constrained_vector_pow_v1f32() {
+; NO-FMA-LABEL: constrained_vector_pow_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    pushq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 16
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    callq powf
+; NO-FMA-NEXT:    popq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 8
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_pow_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    pushq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 16
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    callq powf
+; HAS-FMA-NEXT:    popq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 8
+; HAS-FMA-NEXT:    retq
+entry:
+  %pow = call <1 x float> @llvm.experimental.constrained.pow.v1f32(
+                             <1 x float> <float 42.0>,
+                             <1 x float> <float 3.0>,
+                             metadata !"round.dynamic",
+                             metadata !"fpexcept.strict")
+  ret <1 x float> %pow
+}
+
 define <2 x double> @constrained_vector_pow_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_pow_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    subq $24, %rsp
 ; NO-FMA-NEXT:    .cfi_def_cfa_offset 32
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
 ; NO-FMA-NEXT:    callq pow
%pow = call <4 x double> @llvm.experimental.constrained.pow.v4f64(
                                            double 42.3, double 42.4>,
                              <4 x double> <double 3.0, double 3.0,
                                            double 3.0, double 3.0>,
                              metadata !"round.dynamic",
                              metadata !"fpexcept.strict")
   ret <4 x double> %pow
 }
 
+define <1 x float> @constrained_vector_powi_v1f32() {
+; NO-FMA-LABEL: constrained_vector_powi_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    pushq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 16
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    movl $3, %edi
+; NO-FMA-NEXT:    callq __powisf2
+; NO-FMA-NEXT:    popq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 8
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_powi_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    pushq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 16
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    movl $3, %edi
+; HAS-FMA-NEXT:    callq __powisf2
+; HAS-FMA-NEXT:    popq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 8
+; HAS-FMA-NEXT:    retq
+entry:
+  %powi = call <1 x float> @llvm.experimental.constrained.powi.v1f32(
+                              <1 x float> <float 42.0>,
+                              i32 3,
+                              metadata !"round.dynamic",
+                              metadata !"fpexcept.strict")
+  ret <1 x float> %powi
+}
+
 define <2 x double> @constrained_vector_powi_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_powi_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    subq $24, %rsp
 ; NO-FMA-NEXT:    .cfi_def_cfa_offset 32
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; NO-FMA-NEXT:    movl $3, %edi
 ; NO-FMA-NEXT:    callq __powidf2
%powi = call <4 x double> @llvm.experimental.constrained.powi.v4f64(
                               <4 x double> <double 42.1, double 42.2,
                                             double 42.3, double 42.4>,
                               i32 3,
                               metadata !"round.dynamic",
                               metadata !"fpexcept.strict")
   ret <4 x double> %powi
 }
 
+define <1 x float> @constrained_vector_sin_v1f32() {
+; NO-FMA-LABEL: constrained_vector_sin_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    pushq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 16
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    callq sinf
+; NO-FMA-NEXT:    popq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 8
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_sin_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    pushq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 16
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    callq sinf
+; HAS-FMA-NEXT:    popq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 8
+; HAS-FMA-NEXT:    retq
+entry:
+  %sin = call <1 x float> @llvm.experimental.constrained.sin.v1f32(
+                             <1 x float> <float 42.0>,
+                             metadata !"round.dynamic",
+                             metadata !"fpexcept.strict")
+  ret <1 x float> %sin
+}
 
 define <2 x double> @constrained_vector_sin_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_sin_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    subq $24, %rsp
 ; NO-FMA-NEXT:    .cfi_def_cfa_offset 32
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; NO-FMA-NEXT:    callq sin
entry:
   %sin = call <4 x double> @llvm.experimental.constrained.sin.v4f64(
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
                              metadata !"fpexcept.strict")
   ret <4 x double> %sin
 }
 
+define <1 x float> @constrained_vector_cos_v1f32() {
+; NO-FMA-LABEL: constrained_vector_cos_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    pushq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 16
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    callq cosf
+; NO-FMA-NEXT:    popq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 8
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_cos_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    pushq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 16
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    callq cosf
+; HAS-FMA-NEXT:    popq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 8
+; HAS-FMA-NEXT:    retq
+entry:
+  %cos = call <1 x float> @llvm.experimental.constrained.cos.v1f32(
+                             <1 x float> <float 42.0>,
+                             metadata !"round.dynamic",
+                             metadata !"fpexcept.strict")
+  ret <1 x float> %cos
+}
+
 define <2 x double> @constrained_vector_cos_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_cos_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    subq $24, %rsp
 ; NO-FMA-NEXT:    .cfi_def_cfa_offset 32
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; NO-FMA-NEXT:    callq cos
 ; NO-FMA-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
entry:
   %cos = call <4 x double> @llvm.experimental.constrained.cos.v4f64(
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
                              metadata !"fpexcept.strict")
   ret <4 x double> %cos
 }
 
+define <1 x float> @constrained_vector_exp_v1f32() {
+; NO-FMA-LABEL: constrained_vector_exp_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    pushq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 16
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    callq expf
+; NO-FMA-NEXT:    popq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 8
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_exp_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    pushq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 16
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    callq expf
+; HAS-FMA-NEXT:    popq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 8
+; HAS-FMA-NEXT:    retq
+entry:
+  %exp = call <1 x float> @llvm.experimental.constrained.exp.v1f32(
+                             <1 x float> <float 42.0>,
+                             metadata !"round.dynamic",
+                             metadata !"fpexcept.strict")
+  ret <1 x float> %exp
+}
+
 define <2 x double> @constrained_vector_exp_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_exp_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    subq $24, %rsp
 ; NO-FMA-NEXT:    .cfi_def_cfa_offset 32
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; NO-FMA-NEXT:    callq exp
 ; NO-FMA-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
entry:
   %exp = call <4 x double> @llvm.experimental.constrained.exp.v4f64(
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
                              metadata !"fpexcept.strict")
   ret <4 x double> %exp
 }
 
+define <1 x float> @constrained_vector_exp2_v1f32() {
+; NO-FMA-LABEL: constrained_vector_exp2_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    pushq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 16
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    callq exp2f
+; NO-FMA-NEXT:    popq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 8
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_exp2_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    pushq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 16
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    callq exp2f
+; HAS-FMA-NEXT:    popq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 8
+; HAS-FMA-NEXT:    retq
+entry:
+  %exp2 = call <1 x float> @llvm.experimental.constrained.exp2.v1f32(
+                             <1 x float> <float 42.0>,
+                             metadata !"round.dynamic",
+                             metadata !"fpexcept.strict")
+  ret <1 x float> %exp2
+}
+
 define <2 x double> @constrained_vector_exp2_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_exp2_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    subq $24, %rsp
 ; NO-FMA-NEXT:    .cfi_def_cfa_offset 32
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; NO-FMA-NEXT:    callq exp2
 ; NO-FMA-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
entry:
   %exp2 = call <4 x double> @llvm.experimental.constrained.exp2.v4f64(
                               <4 x double> <double 42.1, double 42.2,
                                             double 42.3, double 42.4>,
                               metadata !"round.dynamic",
                               metadata !"fpexcept.strict")
   ret <4 x double> %exp2
 }
 
+define <1 x float> @constrained_vector_log_v1f32() {
+; NO-FMA-LABEL: constrained_vector_log_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    pushq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 16
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    callq logf
+; NO-FMA-NEXT:    popq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 8
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_log_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    pushq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 16
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    callq logf
+; HAS-FMA-NEXT:    popq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 8
+; HAS-FMA-NEXT:    retq
+entry:
+  %log = call <1 x float> @llvm.experimental.constrained.log.v1f32(
+                             <1 x float> <float 42.0>,
+                             metadata !"round.dynamic",
+                             metadata !"fpexcept.strict")
+  ret <1 x float> %log
+}
+
 define <2 x double> @constrained_vector_log_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_log_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    subq $24, %rsp
 ; NO-FMA-NEXT:    .cfi_def_cfa_offset 32
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; NO-FMA-NEXT:    callq log
 ; NO-FMA-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
entry:
   %log = call <4 x double> @llvm.experimental.constrained.log.v4f64(
                              <4 x double> <double 42.0, double 42.1,
                                            double 42.2, double 42.3>,
                              metadata !"round.dynamic",
                              metadata !"fpexcept.strict")
   ret <4 x double> %log
 }
 
+define <1 x float> @constrained_vector_log10_v1f32() {
+; NO-FMA-LABEL: constrained_vector_log10_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    pushq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 16
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    callq log10f
+; NO-FMA-NEXT:    popq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 8
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_log10_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    pushq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 16
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    callq log10f
+; HAS-FMA-NEXT:    popq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 8
+; HAS-FMA-NEXT:    retq
+entry:
+  %log10 = call <1 x float> @llvm.experimental.constrained.log10.v1f32(
+                             <1 x float> <float 42.0>,
+                             metadata !"round.dynamic",
+                             metadata !"fpexcept.strict")
+  ret <1 x float> %log10
+}
+
 define <2 x double> @constrained_vector_log10_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_log10_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    subq $24, %rsp
 ; NO-FMA-NEXT:    .cfi_def_cfa_offset 32
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; NO-FMA-NEXT:    callq log10
 ; NO-FMA-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
entry:
   %log10 = call <4 x double> @llvm.experimental.constrained.log10.v4f64(
                                <4 x double> <double 42.0, double 42.1,
                                              double 42.2, double 42.3>,
                                metadata !"round.dynamic",
                                metadata !"fpexcept.strict")
   ret <4 x double> %log10
 }
 
+define <1 x float> @constrained_vector_log2_v1f32() {
+; NO-FMA-LABEL: constrained_vector_log2_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    pushq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 16
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    callq log2f
+; NO-FMA-NEXT:    popq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 8
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_log2_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    pushq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 16
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    callq log2f
+; HAS-FMA-NEXT:    popq %rax
+; HAS-FMA-NEXT:    .cfi_def_cfa_offset 8
+; HAS-FMA-NEXT:    retq
+entry:
+  %log2 = call <1 x float> @llvm.experimental.constrained.log2.v1f32(
+                             <1 x float> <float 42.0>,
+                             metadata !"round.dynamic",
+                             metadata !"fpexcept.strict")
+  ret <1 x float> %log2
+}
+
 define <2 x double> @constrained_vector_log2_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_log2_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    subq $24, %rsp
 ; NO-FMA-NEXT:    .cfi_def_cfa_offset 32
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; NO-FMA-NEXT:    callq log2
 ; NO-FMA-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
entry:
   %log2 = call <4 x double> @llvm.experimental.constrained.log2.v4f64(
                               <4 x double> <double 42.0, double 42.1,
                                             double 42.2, double 42.3>,
                               metadata !"round.dynamic",
                               metadata !"fpexcept.strict")
   ret <4 x double> %log2
 }
 
+define <1 x float> @constrained_vector_rint_v1f32() {
+; NO-FMA-LABEL: constrained_vector_rint_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    pushq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 16
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    callq rintf
+; NO-FMA-NEXT:    popq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 8
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_rint_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    vroundss $4, %xmm0, %xmm0, %xmm0
+; HAS-FMA-NEXT:    retq
+entry:
+  %rint = call <1 x float> @llvm.experimental.constrained.rint.v1f32(
+                             <1 x float> <float 42.0>,
+                             metadata !"round.dynamic",
+                             metadata !"fpexcept.strict")
+  ret <1 x float> %rint
+}
+
 define <2 x double> @constrained_vector_rint_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_rint_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    subq $24, %rsp
 ; NO-FMA-NEXT:    .cfi_def_cfa_offset 32
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; NO-FMA-NEXT:    callq rint
 ; NO-FMA-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
entry:
   %rint = call <4 x double> @llvm.experimental.constrained.rint.v4f64(
                         <4 x double> <double 42.1, double 42.2,
                                       double 42.3, double 42.4>,
                         metadata !"round.dynamic",
                         metadata !"fpexcept.strict")
   ret <4 x double> %rint
 }
 
+define <1 x float> @constrained_vector_nearbyint_v1f32() {
+; NO-FMA-LABEL: constrained_vector_nearbyint_v1f32:
+; NO-FMA:       # %bb.0: # %entry
+; NO-FMA-NEXT:    pushq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 16
+; NO-FMA-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; NO-FMA-NEXT:    callq nearbyintf
+; NO-FMA-NEXT:    popq %rax
+; NO-FMA-NEXT:    .cfi_def_cfa_offset 8
+; NO-FMA-NEXT:    retq
+;
+; HAS-FMA-LABEL: constrained_vector_nearbyint_v1f32:
+; HAS-FMA:       # %bb.0: # %entry
+; HAS-FMA-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; HAS-FMA-NEXT:    vroundss $12, %xmm0, %xmm0, %xmm0
+; HAS-FMA-NEXT:    retq
+entry:
+  %nearby = call <1 x float> @llvm.experimental.constrained.nearbyint.v1f32(
+                               <1 x float> <float 42.0>,
+                               metadata !"round.dynamic",
+                               metadata !"fpexcept.strict")
+  ret <1 x float> %nearby
+}
+
 define <2 x double> @constrained_vector_nearbyint_v2f64() {
 ; NO-FMA-LABEL: constrained_vector_nearbyint_v2f64:
 ; NO-FMA:       # %bb.0: # %entry
 ; NO-FMA-NEXT:    subq $24, %rsp
 ; NO-FMA-NEXT:    .cfi_def_cfa_offset 32
 ; NO-FMA-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; NO-FMA-NEXT:    callq nearbyint
 ; NO-FMA-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill

 declare <2 x double> @llvm.experimental.constrained.exp.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.exp2.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.log.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.log10.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.log2.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.rint.v2f64(<2 x double>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.nearbyint.v2f64(<2 x double>, metadata, metadata)
 
+; Scalar width declarations
+declare <1 x float> @llvm.experimental.constrained.fdiv.v1f32(<1 x float>, <1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.fmul.v1f32(<1 x float>, <1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.fadd.v1f32(<1 x float>, <1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.fsub.v1f32(<1 x float>, <1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.fma.v1f32(<1 x float>, <1 x float>, <1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.sqrt.v1f32(<1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.pow.v1f32(<1 x float>, <1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.powi.v1f32(<1 x float>, i32, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.sin.v1f32(<1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.cos.v1f32(<1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.exp.v1f32(<1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.exp2.v1f32(<1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.log.v1f32(<1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.log10.v1f32(<1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.log2.v1f32(<1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.rint.v1f32(<1 x float>, metadata, metadata)
+declare <1 x float> @llvm.experimental.constrained.nearbyint.v1f32(<1 x float>, metadata, metadata)
+
 ; Illegal width declarations
 declare <3 x float> @llvm.experimental.constrained.fdiv.v3f32(<3 x float>, <3 x float>, metadata, metadata)
 declare <3 x double> @llvm.experimental.constrained.fdiv.v3f64(<3 x double>, <3 x double>, metadata, metadata)
 declare <3 x float> @llvm.experimental.constrained.fmul.v3f32(<3 x float>, <3 x float>, metadata, metadata)
 declare <3 x double> @llvm.experimental.constrained.fmul.v3f64(<3 x double>, <3 x double>, metadata, metadata)
 declare <3 x float> @llvm.experimental.constrained.fadd.v3f32(<3 x float>, <3 x float>, metadata, metadata)
 declare <3 x double> @llvm.experimental.constrained.fadd.v3f64(<3 x double>, <3 x double>, metadata, metadata)
 declare <3 x float> @llvm.experimental.constrained.fsub.v3f32(<3 x float>, <3 x float>, metadata, metadata)