[LoopVectorizer] Take into account call register pressure when selecting interleave count

[rob.lougher]

Summary

This is a follow up patch to D48193. In the previous review an example of poor code generation was given, where the interleave count caused a loop containing a veclib call to generate excessive spills/reloads. The suggested fix was to simply use an IC of 1 when a veclib call is present.

Rather than use an IC of 1 for all platforms, this patch:

1. Adds a call register pressure calculation.
2. Adds calls to TargetTransformInfo to obtain information about the number of preserved registers.

At present only a basic implementation of the TTI calls is provided for x86. However, the default implementation means that this change will have no affect on other platforms.

[grandinj]

lib/Target/X86/X86TargetTransformInfo.cpp

191

perhaps an

assert(hasNumberOfCallPreservedRegisters() && "you need to call hasNumberOfCallPreservedRegisters before calling this")

would be a good idea here, to catch incorrect usage?

[rob.lougher]

lib/Target/X86/X86TargetTransformInfo.cpp

191

Yes, excellent idea. I'll add an assert to the function in TargetTransformInfo.cpp, before calling the TTIImpl.

[rob.lougher]

Added asserts to TTI functions to catch misuse.

include/llvm/Analysis/TargetTransformInfo.h

public:
   unsigned getMinimumVF(unsigned ElemWidth) const;
 
   /// \return True if it should be considered for address type promotion.
   /// \p AllowPromotionWithoutCommonHeader Set true if promoting \p I is
   /// profitable without finding other extensions fed by the same input.
   bool shouldConsiderAddressTypePromotion(
       const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const;
 
+  /// \return True if the number of call preserved registers is available
+  /// for the target. This is the case when the target has a single class
+  /// of vector register, or when the number of preserved registers is the
+  /// same (e.g. the x86-64 System V ABI, where no XMM, YMM or ZMM registers
+  /// are preserved). If the ABI preserves different numbers depending on
+  /// the class it can't be handled without tracking in which class of
+  /// register a value will be stored. For example, Win64 preserves XMM8-XMM15.
+  /// If the target supports AVX, a value may be in either a XMM or a YMM
+  /// register, and the ABI cannot be handled. However, if the target does
+  /// not support AVX, a vector value can only be in a XMM register and a
+  /// value can be returned for the ABI.
+  bool hasNumberOfCallPreservedRegisters(bool Vector) const;
+
+  /// \return The number of scalar or vector registers preserved across a
+  /// call. The returned value only applies when hasNumberOfPreservedRegisters
+  /// returns true.
+  unsigned getNumberOfCallPreservedRegisters(bool Vector) const;
+
+  /// \return The maximum interleave factor that should be used for the
+  /// target when a loop contains a call. By default, the target
+  /// getMaxInterleaveFactor value is used. The returned value only applies
+  /// when hasNumberOfPreservedRegisters returns false.
+  unsigned getMaxCallInterleaveFactor(unsigned VF) const;
+
   /// \return The size of a cache line in bytes.
   unsigned getCacheLineSize() const;
 
   /// The possible cache levels
   enum class CacheLevel {
     L1D,   // The L1 data cache
     L2D,   // The L2 data cache
 
public:
   virtual int getIntImmCodeSizeCost(unsigned Opc, unsigned Idx, const APInt &Imm,
                                     Type *Ty) = 0;
   virtual int getIntImmCost(const APInt &Imm, Type *Ty) = 0;
   virtual int getIntImmCost(unsigned Opc, unsigned Idx, const APInt &Imm,
                             Type *Ty) = 0;
   virtual int getIntImmCost(Intrinsic::ID IID, unsigned Idx, const APInt &Imm,
                             Type *Ty) = 0;
   virtual unsigned getNumberOfRegisters(bool Vector) = 0;
+  virtual bool hasNumberOfCallPreservedRegisters(bool Vector) = 0;
+  virtual unsigned getNumberOfCallPreservedRegisters(bool Vector) = 0;
+  virtual unsigned getMaxCallInterleaveFactor(unsigned VF) = 0;
   virtual unsigned getRegisterBitWidth(bool Vector) const = 0;
   virtual unsigned getMinVectorRegisterBitWidth() = 0;
   virtual bool shouldMaximizeVectorBandwidth(bool OptSize) const = 0;
   virtual unsigned getMinimumVF(unsigned ElemWidth) const = 0;
   virtual bool shouldConsiderAddressTypePromotion(
       const Instruction &I, bool &AllowPromotionWithoutCommonHeader) = 0;
   virtual unsigned getCacheLineSize() = 0;
   virtual llvm::Optional<unsigned> getCacheSize(CacheLevel Level) = 0;
public:
   }
   int getIntImmCost(Intrinsic::ID IID, unsigned Idx, const APInt &Imm,
                     Type *Ty) override {
     return Impl.getIntImmCost(IID, Idx, Imm, Ty);
   }
   unsigned getNumberOfRegisters(bool Vector) override {
     return Impl.getNumberOfRegisters(Vector);
   }
+  bool hasNumberOfCallPreservedRegisters(bool Vector) override {
+    return Impl.hasNumberOfCallPreservedRegisters(Vector);
+  }
+  unsigned getNumberOfCallPreservedRegisters(bool Vector) override {
+    return Impl.getNumberOfCallPreservedRegisters(Vector);
+  }
+  unsigned getMaxCallInterleaveFactor(unsigned VF) override {
+    return Impl.getMaxCallInterleaveFactor(VF);
+  }
   unsigned getRegisterBitWidth(bool Vector) const override {
     return Impl.getRegisterBitWidth(Vector);
   }
   unsigned getMinVectorRegisterBitWidth() override {
     return Impl.getMinVectorRegisterBitWidth();
   }
   bool shouldMaximizeVectorBandwidth(bool OptSize) const override {
     return Impl.shouldMaximizeVectorBandwidth(OptSize);

include/llvm/Analysis/TargetTransformInfoImpl.h

unsigned getIntImmCost(Intrinsic::ID IID, unsigned Idx, const APInt &Imm,
                          Type *Ty) {
     return TTI::TCC_Free;
   }
 
   unsigned getNumberOfRegisters(bool Vector) { return 8; }
 
   unsigned getRegisterBitWidth(bool Vector) const { return 32; }
 
+  bool hasNumberOfCallPreservedRegisters(bool Vector) { return false; };
+
+  unsigned getNumberOfCallPreservedRegisters(bool Vector) { return 0; }
+
+  unsigned getMaxCallInterleaveFactor(unsigned VF) { return 1; }
+
   unsigned getMinVectorRegisterBitWidth() { return 128; }
 
   bool shouldMaximizeVectorBandwidth(bool OptSize) const { return false; }
 
   unsigned getMinimumVF(unsigned ElemWidth) const { return 0; }
 
   bool
   shouldConsiderAddressTypePromotion(const Instruction &I,

include/llvm/CodeGen/BasicTTIImpl.h

public:
 
   /// \name Vector TTI Implementations
   /// @{
 
   unsigned getNumberOfRegisters(bool Vector) { return Vector ? 0 : 1; }
 
   unsigned getRegisterBitWidth(bool Vector) const { return 32; }
 
+  unsigned getMaxCallInterleaveFactor(unsigned VF) {
+    // By default the maximum call interleave factor is the target's
+    // maximum interleave factor (i.e. calls have no affect).
+    return static_cast<T *>(this)->getMaxInterleaveFactor(VF);
+  }
+
   /// Estimate the overhead of scalarizing an instruction. Insert and Extract
   /// are set if the result needs to be inserted and/or extracted from vectors.
   unsigned getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) {
     assert(Ty->isVectorTy() && "Can only scalarize vectors");
     unsigned Cost = 0;
 
     for (int i = 0, e = Ty->getVectorNumElements(); i < e; ++i) {
       if (Insert)

lib/Analysis/TargetTransformInfo.cpp

int TargetTransformInfo::getIntImmCost(Intrinsic::ID IID, unsigned Idx,
   assert(Cost >= 0 && "TTI should not produce negative costs!");
   return Cost;
 }
 
 unsigned TargetTransformInfo::getNumberOfRegisters(bool Vector) const {
   return TTIImpl->getNumberOfRegisters(Vector);
 }
 
+bool TargetTransformInfo::hasNumberOfCallPreservedRegisters(bool Vector) const {
+  return TTIImpl->hasNumberOfCallPreservedRegisters(Vector);
+}
+
+unsigned
+TargetTransformInfo::getNumberOfCallPreservedRegisters(bool Vector) const {
+  assert(
+      TTIImpl->hasNumberOfCallPreservedRegisters(Vector) &&
+      "Shouldn't be called when hasNumberOfPreservedRegisters() returns false");
+  return TTIImpl->getNumberOfCallPreservedRegisters(Vector);
+}
+
+unsigned TargetTransformInfo::getMaxCallInterleaveFactor(unsigned VF) const {
+  assert(
+      !TTIImpl->hasNumberOfCallPreservedRegisters(VF > 1) &&
+      "Shouldn't be called when hasNumberOfPreservedRegisters() returns true");
+  return TTIImpl->getMaxCallInterleaveFactor(VF);
+}
+
 unsigned TargetTransformInfo::getRegisterBitWidth(bool Vector) const {
   return TTIImpl->getRegisterBitWidth(Vector);
 }
 
 unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const {
   return TTIImpl->getMinVectorRegisterBitWidth();
 }
 

lib/Target/X86/X86TargetTransformInfo.h

public:
 
   /// \name Vector TTI Implementations
   /// @{
 
   unsigned getNumberOfRegisters(bool Vector);
   unsigned getRegisterBitWidth(bool Vector) const;
   unsigned getLoadStoreVecRegBitWidth(unsigned AS) const;
   unsigned getMaxInterleaveFactor(unsigned VF);
+  bool hasNumberOfCallPreservedRegisters(bool Vector);
+  unsigned getNumberOfCallPreservedRegisters(bool Vector);
   int getArithmeticInstrCost(
       unsigned Opcode, Type *Ty,
       TTI::OperandValueKind Opd1Info = TTI::OK_AnyValue,
       TTI::OperandValueKind Opd2Info = TTI::OK_AnyValue,
       TTI::OperandValueProperties Opd1PropInfo = TTI::OP_None,
       TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None,
       ArrayRef<const Value *> Args = ArrayRef<const Value *>());
   int getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, Type *SubTp);

lib/Target/X86/X86TargetTransformInfo.cpp

unsigned X86TTIImpl::getMaxInterleaveFactor(unsigned VF) {
   // Sandybridge and Haswell have multiple execution ports and pipelined
   // vector units.
   if (ST->hasAVX())
     return 4;
 
   return 2;
 }
 
+bool X86TTIImpl::hasNumberOfCallPreservedRegisters(bool Vector) {
+  if (!Vector || !ST->is64Bit())
+    return false;
+
+  // Win64 ABI preserves XMM8-XMM15. If the target does not support
+  // AVX, then a vector value must be in a XMM register. Otherwise
+  // the value could be in a YMM or ZMM register (in which case it
+  // will not be preserved).
+  if (ST->isOSWindows() && !ST->hasAVX())
+    return true;
+
+  // If the target uses the System V ABI, no vector registers are
+  // preserved.
+  return (ST->isTargetDarwin() || ST->isTargetLinux() ||
+          ST->isTargetSolaris() || ST->isTargetKFreeBSD() ||
+          ST->isTargetPS4());
+}
+
+unsigned X86TTIImpl::getNumberOfCallPreservedRegisters(bool Vector) {
+  if (ST->isOSWindows())
+    return 8;
+
+  return 0;
+}
+
 int X86TTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty,
     TTI::OperandValueKind Op1Info, TTI::OperandValueKind Op2Info,
     TTI::OperandValueProperties Opd1PropInfo,
     TTI::OperandValueProperties Opd2PropInfo,
     ArrayRef<const Value *> Args) {
   // Legalize the type.
   std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);

lib/Transforms/Vectorize/LoopVectorize.cpp

public:
   /// A struct that represents some properties of the register usage
   /// of a loop.
   struct RegisterUsage {
     /// Holds the number of loop invariant values that are used in the loop.
     unsigned LoopInvariantRegs;
 
     /// Holds the maximum number of concurrent live intervals in the loop.
     unsigned MaxLocalUsers;
+
+    /// Holds the maximum number of live registers at a callsite in the loop.
+    unsigned MaxCallUsers;
+
+    /// Holds the maximum interleave factor for a call in the loop.
+    unsigned MaxCallIC;
   };
 
   /// \return Returns information about the register usages of the loop for the
   /// given vectorization factors.
-  SmallVector<RegisterUsage, 8> calculateRegisterUsage(ArrayRef<unsigned> VFs);
+  SmallVector<RegisterUsage, 8> calculateRegisterUsage(ArrayRef<unsigned> VFs,
+                                                     bool handleCalls = false);
 
   /// Collect values we want to ignore in the cost model.
   void collectValuesToIgnore();
 
   /// \returns The smallest bitwidth each instruction can be represented with.
   /// The vector equivalents of these instructions should be truncated to this
   /// type.
   const MapVector<Instruction *, uint64_t> &getMinimalBitwidths() const {
static unsigned getVectorIntrinsicCost(CallInst *CI, unsigned VF,
   return TTI.getIntrinsicInstrCost(ID, CI->getType(), Operands, FMF, VF);
 }
 
 static Type *smallestIntegerVectorType(Type *T1, Type *T2) {
   auto *I1 = cast<IntegerType>(T1->getVectorElementType());
   auto *I2 = cast<IntegerType>(T2->getVectorElementType());
   return I1->getBitWidth() < I2->getBitWidth() ? T1 : T2;
 }
+
 static Type *largestIntegerVectorType(Type *T1, Type *T2) {
   auto *I1 = cast<IntegerType>(T1->getVectorElementType());
   auto *I2 = cast<IntegerType>(T2->getVectorElementType());
   return I1->getBitWidth() > I2->getBitWidth() ? T1 : T2;
 }
 
 void InnerLoopVectorizer::truncateToMinimalBitwidths() {
   // For every instruction `I` in MinBWs, truncate the operands, create a
unsigned LoopVectorizationCostModel::selectInterleaveCount(bool OptForSize,
   if (VF == 1) {
     if (ForceTargetNumScalarRegs.getNumOccurrences() > 0)
       TargetNumRegisters = ForceTargetNumScalarRegs;
   } else {
     if (ForceTargetNumVectorRegs.getNumOccurrences() > 0)
       TargetNumRegisters = ForceTargetNumVectorRegs;
   }
 
-  RegisterUsage R = calculateRegisterUsage({VF})[0];
+  RegisterUsage R = calculateRegisterUsage({VF}, true)[0];
   // We divide by these constants so assume that we have at least one
   // instruction that uses at least one register.
   R.MaxLocalUsers = std::max(R.MaxLocalUsers, 1U);
 
   // We calculate the interleave count using the following formula.
   // Subtract the number of loop invariants from the number of available
   // registers. These registers are used by all of the interleaved instances.
   // Next, divide the remaining registers by the number of registers that is
   // required by the loop, in order to estimate how many parallel instances
   // fit without causing spills. All of this is rounded down if necessary to be
   // a power of two. We want power of two interleave count to simplify any
   // addressing operations or alignment considerations.
   unsigned IC = PowerOf2Floor((TargetNumRegisters - R.LoopInvariantRegs) /
                               R.MaxLocalUsers);
 
   // Don't count the induction variable as interleaved.
   if (EnableIndVarRegisterHeur)
     IC = PowerOf2Floor((TargetNumRegisters - R.LoopInvariantRegs - 1) /
                        std::max(1U, (R.MaxLocalUsers - 1)));
 
+  // If the loop contains a call with live values at the callsite, use an IC
+  // no greater than the maximum call IC (rounded down to a power of 2). An
+  // IC above this will cause additional spilling.
+  if (R.MaxCallUsers > 0)
+    IC = std::min(IC, (unsigned)PowerOf2Floor(R.MaxCallIC));
+
   // Clamp the interleave ranges to reasonable counts.
   unsigned MaxInterleaveCount = TTI.getMaxInterleaveFactor(VF);
 
   // Check if the user has overridden the max.
   if (VF == 1) {
     if (ForceTargetMaxScalarInterleaveFactor.getNumOccurrences() > 0)
       MaxInterleaveCount = ForceTargetMaxScalarInterleaveFactor;
   } else {
if (TTI.enableAggressiveInterleaving(HasReductions)) {
     return IC;
   }
 
   LLVM_DEBUG(dbgs() << "LV: Not Interleaving.\n");
   return 1;
 }
 
 SmallVector<LoopVectorizationCostModel::RegisterUsage, 8>
-LoopVectorizationCostModel::calculateRegisterUsage(ArrayRef<unsigned> VFs) {
+LoopVectorizationCostModel::calculateRegisterUsage(ArrayRef<unsigned> VFs,
+                                                   bool handleCalls) {
   // This function calculates the register usage by measuring the highest number
   // of values that are alive at a single location. Obviously, this is a very
   // rough estimation. We scan the loop in a topological order in order and
   // assign a number to each instruction. We use RPO to ensure that defs are
   // met before their users. We assume that each instruction that has in-loop
   // users starts an interval. We record every time that an in-loop value is
   // used, so we have a list of the first and last occurrences of each
   // instruction. Next, we transpose this data structure into a multi map that
LoopVectorizationCostModel::calculateRegisterUsage(ArrayRef<unsigned> VFs,
   if (Legal->getMaxSafeDepDistBytes() != -1U)
     MaxSafeDepDist = Legal->getMaxSafeDepDistBytes() * 8;
   unsigned WidestRegister =
       std::min(TTI.getRegisterBitWidth(true), MaxSafeDepDist);
   const DataLayout &DL = TheFunction->getParent()->getDataLayout();
 
   SmallVector<RegisterUsage, 8> RUs(VFs.size());
   SmallVector<unsigned, 8> MaxUsages(VFs.size(), 0);
+  SmallVector<unsigned, 8> MaxCallUsages(VFs.size(), 0);
+  SmallVector<unsigned, 8> MaxCallIC(VFs.size(), -1);
 
   LLVM_DEBUG(dbgs() << "LV(REG): Calculating max register usage:\n");
 
   // A lambda that gets the register usage for the given type and VF.
   auto GetRegUsage = [&DL, WidestRegister](Type *Ty, unsigned VF) {
     if (Ty->isTokenTy())
       return 0U;
     unsigned TypeSize = DL.getTypeSizeInBits(Ty->getScalarType());
if (!Ends.count(I))
       continue;
 
     // Skip ignored values.
     if (ValuesToIgnore.count(I))
       continue;
 
     // For each VF find the maximum usage of registers.
     for (unsigned j = 0, e = VFs.size(); j < e; ++j) {
-      if (VFs[j] == 1) {
-        MaxUsages[j] = std::max(MaxUsages[j], OpenIntervals.size());
-        continue;
-      }
+      unsigned RegUsage = 0;
+      if (VFs[j] == 1)
+        RegUsage = OpenIntervals.size();
+      else {
         collectUniformsAndScalars(VFs[j]);
         // Count the number of live intervals.
-      unsigned RegUsage = 0;
         for (auto Inst : OpenIntervals) {
           // Skip ignored values for VF > 1.
           if (VecValuesToIgnore.count(Inst) ||
               isScalarAfterVectorization(Inst, VFs[j]))
             continue;
           RegUsage += GetRegUsage(Inst->getType(), VFs[j]);
         }
+      }
       MaxUsages[j] = std::max(MaxUsages[j], RegUsage);
+
+      // If the instruction is a call calculate the register pressure.
+      if (CallInst *CI = dyn_cast<CallInst>(I)) {
+        if (!handleCalls)
+          continue;
+
+        MaxCallUsages[j] = std::max(MaxCallUsages[j], RegUsage);
+
+        if (!TTI.hasNumberOfCallPreservedRegisters(VFs[j] > 1)) {
+          MaxCallIC[j] = TTI.getMaxCallInterleaveFactor(VFs[j]);
+          continue;
+        }
+
+        if (RegUsage == 0)
+          continue;
+
+        Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
+        bool NeedToScalarize;
+        unsigned CallCost = getVectorCallCost(CI, VFs[j], TTI, TLI,
+            NeedToScalarize);
+
+        // If the vector intrinsic is cheaper than the vector call
+        // assume it will be expanded, leaving no call (if it is the
+        // same cost it is likely to be scalarized).
+        if (ID && getVectorIntrinsicCost(CI, VFs[j], TTI, TLI) < CallCost)
+          continue;
+
+        // Calculate the number of live values that are not used after the
+        // call. This is the number of values that "end" at the next
+        // instruction.
+        unsigned DeadAfterCall = 0;
+
+        if ((i + 1) == Index)
+          DeadAfterCall = RegUsage;
+        else {
+          InstrList &List = TransposeEnds[i + 1];
+          if (VFs[j] == 1)
+            DeadAfterCall = List.size();
+          else
+            for (Instruction *Inst : List)
+              DeadAfterCall += GetRegUsage(Inst->getType(), VFs[j]);
+        }
+
+        unsigned RetUsage = VFs[j] == 1 ? 1 : GetRegUsage(I->getType(), VFs[j]);
+
+        // The interleave factor for the call is the number of times the call
+        // and live values at the callsite can be cloned without causing
+        // additional spilling. For the live values, the number is simply
+        // the number of preserved registers across the call, divided by the
+        // number of live values. However, the calculation must also take into
+        // account the additional registers needed for the return values of
+        // the cloned calls (the return value for the last call will not need
+        // to be preserved). Similarly, some of the live values may not be
+        // used after the call (i.e. call arguments). When the additional
+        // registers for the return equals the number of registers dead after
+        // the call they cancel each other out.
+        unsigned PR = TTI.getNumberOfCallPreservedRegisters(VFs[j] > 1);
+        unsigned Diff = DeadAfterCall > RetUsage ? 0 : RetUsage - DeadAfterCall;
+        MaxCallIC[j] = std::min(MaxCallIC[j], (PR + Diff) / (RegUsage + Diff));
+      }
     }
 
     LLVM_DEBUG(dbgs() << "LV(REG): At #" << i << " Interval # "
                       << OpenIntervals.size() << '\n');
 
     // Add the current instruction to the list of open intervals.
     OpenIntervals.insert(I);
   }
for (unsigned i = 0, e = VFs.size(); i < e; ++i) {
 
     LLVM_DEBUG(dbgs() << "LV(REG): VF = " << VFs[i] << '\n');
     LLVM_DEBUG(dbgs() << "LV(REG): Found max usage: " << MaxUsages[i] << '\n');
     LLVM_DEBUG(dbgs() << "LV(REG): Found invariant usage: " << Invariant
                       << '\n');
 
     RU.LoopInvariantRegs = Invariant;
     RU.MaxLocalUsers = MaxUsages[i];
+
+    if (handleCalls) {
+      LLVM_DEBUG(dbgs() << "LV(REG): Found max call usage: "
+                        << MaxCallUsages[i] << '\n');
+      LLVM_DEBUG(dbgs() << "LV(REG): Found max call IC: "
+                        << MaxCallIC[i] << '\n');
+      RU.MaxCallUsers = MaxCallUsages[i];
+      RU.MaxCallIC = MaxCallIC[i];
+    }
+
     RUs[i] = RU;
   }
 
   return RUs;
 }
 
 bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(Instruction *I){
   // TODO: Cost model for emulated masked load/store is completely

test/Transforms/LoopVectorize/X86/interleaving-veclib-call.ll

+; RUN: opt -S -mtriple=x86_64-unknown-linux -mcpu=btver2 -vector-library=SVML -loop-vectorize < %s | FileCheck %s
+
+; This test checks that when a call is vectorized with a vector library call
+; the interleave count used is 1 (i.e. the call appears only once).  As loops
+; with reductions are treated specially by the cost model we also test this
+; case. Finally, a test is included that checks that no restriction of the
+; interleave count is done when the call is not vectorized to a vector library
+; call.
+
+; CHECK-LABEL: test
+; CHECK: call <8 x float> @__svml_sinf8
+; CHECK-NOT: call <8 x float> @__svml_sinf8
+
+define void @sinf-test(float* nocapture readonly %a, float* nocapture %b, i32 %n) {
+entry:
+  %cmp7 = icmp sgt i32 %n, 0
+  br i1 %cmp7, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @sinf(float %0)
+  %arrayidx2 = getelementptr inbounds float, float* %b, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+declare dso_local float @sinf(float) local_unnamed_addr
+
+; CHECK-LABEL: sinf-reduc-test
+; CHECK: call fast <8 x float> @__svml_sinf8
+; CHECK-NOT: call fast <8 x float> @__svml_sinf8
+
+define float @sinf-reduc-test(float* nocapture readonly %a, i32 %n) {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %s.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]
+  ret float %s.0.lcssa
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %s.07 = phi float [ 0.000000e+00, %for.body.preheader ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %1 = tail call fast float @llvm.sin.f32(float %0)
+  %add = fadd fast float %1, %s.07
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+declare float @llvm.sin.f32(float)
+
+; CHECK-LABEL: ceilf-test
+; CHECK: call <8 x float> @llvm.ceil.v8f32
+; CHECK: call <8 x float> @llvm.ceil.v8f32
+; CHECK: call <8 x float> @llvm.ceil.v8f32
+; CHECK: call <8 x float> @llvm.ceil.v8f32
+
+define void @ceilf-test(float* nocapture readonly %a, float* nocapture %b, i32 %n) {
+entry:
+  %cmp7 = icmp sgt i32 %n, 0
+  br i1 %cmp7, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %1 = tail call float @llvm.ceil.f32(float %0)
+  %arrayidx2 = getelementptr inbounds float, float* %b, i64 %indvars.iv
+  store float %1, float* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+declare float @llvm.ceil.f32(float)