[LV][LoopUtils] Add UseReductionIntrinsic to createTargetReduction

[dmgreen]

Summary

This changes the interface to createTargetReduction and createSimpleTargetReduction to take a UseReductionIntrinsic bool, instead of asking TTI. This allows us to calculate UseReductionIntrinsic earlier in the vectorizer and reduce the need to propagate TTI into RecuctionRecipes.

In the process I added a getReductionFlags mathod for conveniently creating the TargetTransformInfo::ReductionFlags flags used by several methods.

[dmgreen]

Rebase

[dmgreen]

ping

llvm/include/llvm/Analysis/IVDescriptors.h

 
 #ifndef LLVM_ANALYSIS_IVDESCRIPTORS_H
 #define LLVM_ANALYSIS_IVDESCRIPTORS_H
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Support/Casting.h"
 
 namespace llvm {
 
public:
   /// Returns true if all source operands of the recurrence are SExtInsts.
   bool isSigned() { return IsSigned; }
 
   /// Attempts to find a chain of operations from Phi to LoopExitInst that can
   /// be treated as a set of reductions instructions for in-loop reductions.
   SmallVector<Instruction *, 4> getReductionOpChain(PHINode *Phi,
                                                     Loop *L) const;
 
+  /// Return the correct TargetTransformInfo::ReductionFlags for this reduction,
+  /// with NoNaN set as per the argument.
+  TargetTransformInfo::ReductionFlags getReductionFlags(bool NoNaN);
+
 private:
   // The starting value of the recurrence.
   // It does not have to be zero!
   TrackingVH<Value> StartValue;
   // The instruction who's value is used outside the loop.
   Instruction *LoopExitInstr = nullptr;
   // The kind of the recurrence.
   RecurrenceKind Kind = RK_NoRecurrence;

llvm/include/llvm/Transforms/Utils/LoopUtils.h

 Value *getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op,
                            RecurrenceDescriptor::MinMaxRecurrenceKind
                                MinMaxKind = RecurrenceDescriptor::MRK_Invalid,
                            ArrayRef<Value *> RedOps = None);
 
 /// Create a target reduction of the given vector. The reduction operation
 /// is described by the \p Opcode parameter. min/max reductions require
 /// additional information supplied in \p Flags.
-/// The target is queried to determine if intrinsics or shuffle sequences are
-/// required to implement the reduction.
+/// UseReductionIntrinsic is used to determine if intrinsics or shuffle
+/// sequences are required to implement the reduction.
 /// Fast-math-flags are propagated using the IRBuilder's setting.
-Value *createSimpleTargetReduction(IRBuilderBase &B,
-                                   const TargetTransformInfo *TTI,
-                                   unsigned Opcode, Value *Src,
+Value *createSimpleTargetReduction(IRBuilderBase &B, unsigned Opcode,
+                                   Value *Src, bool UseReductionIntrinsic,
                                    TargetTransformInfo::ReductionFlags Flags =
                                        TargetTransformInfo::ReductionFlags(),
                                    ArrayRef<Value *> RedOps = None);
 
 /// Create a generic target reduction using a recurrence descriptor \p Desc
-/// The target is queried to determine if intrinsics or shuffle sequences are
-/// required to implement the reduction.
+/// UseReductionIntrinsic is used to determine if intrinsics or shuffle
+/// sequences are required to implement the reduction.
 /// Fast-math-flags are propagated using the RecurrenceDescriptor.
-Value *createTargetReduction(IRBuilderBase &B, const TargetTransformInfo *TTI,
-                             RecurrenceDescriptor &Desc, Value *Src,
+Value *createTargetReduction(IRBuilderBase &B, RecurrenceDescriptor &Desc,
+                             Value *Src, bool UseReductionIntrinsic,
                              bool NoNaN = false);
 
 /// Get the intersection (logical and) of all of the potential IR flags
 /// of each scalar operation (VL) that will be converted into a vector (I).
 /// If OpValue is non-null, we only consider operations similar to OpValue
 /// when intersecting.
 /// Flag set: NSW, NUW, exact, and all of fast-math.
 void propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue = nullptr);

llvm/lib/Analysis/IVDescriptors.cpp

while (Cur != LoopExitInstr) {
     ReductionOperations.push_back(Cur);
     Cur = getNextInstruction(Cur);
   }
 
   ReductionOperations.push_back(Cur);
   return ReductionOperations;
 }
 
+TargetTransformInfo::ReductionFlags
+RecurrenceDescriptor::getReductionFlags(bool NoNaN) {
+  RecurrenceKind RecKind = getRecurrenceKind();
+  TargetTransformInfo::ReductionFlags Flags;
+  Flags.NoNaN = NoNaN;
+
+  switch (RecKind) {
+  case RK_IntegerMinMax: {
+    MinMaxRecurrenceKind MMKind = getMinMaxRecurrenceKind();
+    Flags.IsMaxOp = (MMKind == MRK_SIntMax || MMKind == MRK_UIntMax);
+    Flags.IsSigned = (MMKind == MRK_SIntMax || MMKind == MRK_SIntMin);
+    break;
+  }
+  case RK_FloatMinMax: {
+    Flags.IsMaxOp = getMinMaxRecurrenceKind() == MRK_FloatMax;
+    break;
+  }
+  default:
+    break;
+  }
+
+  return Flags;
+}
+
 InductionDescriptor::InductionDescriptor(Value *Start, InductionKind K,
                                          const SCEV *Step, BinaryOperator *BOp,
                                          SmallVectorImpl<Instruction *> *Casts)
     : StartValue(Start), IK(K), Step(Step), InductionBinOp(BOp) {
   assert(IK != IK_NoInduction && "Not an induction");
 
   // Start value type should match the induction kind and the value
   // itself should not be null.

llvm/lib/Transforms/Utils/LoopUtils.cpp

if (auto *ReductionInst = dyn_cast<Instruction>(TmpVec))
       ReductionInst->dropPoisonGeneratingFlags();
   }
   // The result is in the first element of the vector.
   return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
 }
 
 /// Create a simple vector reduction specified by an opcode and some
 /// flags (if generating min/max reductions).
-Value *llvm::createSimpleTargetReduction(
-    IRBuilderBase &Builder, const TargetTransformInfo *TTI, unsigned Opcode,
-    Value *Src, TargetTransformInfo::ReductionFlags Flags,
+Value *
+llvm::createSimpleTargetReduction(IRBuilderBase &Builder, unsigned Opcode,
+                                  Value *Src, bool UseReductionIntrinsic,
+                                  TargetTransformInfo::ReductionFlags Flags,
                                   ArrayRef<Value *> RedOps) {
   auto *SrcVTy = cast<VectorType>(Src->getType());
 
   std::function<Value *()> BuildFunc;
   using RD = RecurrenceDescriptor;
   RD::MinMaxRecurrenceKind MinMaxKind = RD::MRK_Invalid;
 
   switch (Opcode) {
   case Instruction::Add:
if (Flags.IsMaxOp) {
       MinMaxKind = RD::MRK_FloatMin;
       BuildFunc = [&]() { return Builder.CreateFPMinReduce(Src, Flags.NoNaN); };
     }
     break;
   default:
     llvm_unreachable("Unhandled opcode");
     break;
   }
-  if (ForceReductionIntrinsic ||
-      TTI->useReductionIntrinsic(Opcode, Src->getType(), Flags))
+  if (ForceReductionIntrinsic || UseReductionIntrinsic)
     return BuildFunc();
   return getShuffleReduction(Builder, Src, Opcode, MinMaxKind, RedOps);
 }
 
 /// Create a vector reduction using a given recurrence descriptor.
-Value *llvm::createTargetReduction(IRBuilderBase &B,
-                                   const TargetTransformInfo *TTI,
-                                   RecurrenceDescriptor &Desc, Value *Src,
+Value *llvm::createTargetReduction(IRBuilderBase &B, RecurrenceDescriptor &Desc,
+                                   Value *Src, bool UseReductionIntrinsic,
                                    bool NoNaN) {
   // TODO: Support in-order reductions based on the recurrence descriptor.
   using RD = RecurrenceDescriptor;
   RD::RecurrenceKind RecKind = Desc.getRecurrenceKind();
-  TargetTransformInfo::ReductionFlags Flags;
-  Flags.NoNaN = NoNaN;
+  TargetTransformInfo::ReductionFlags Flags = Desc.getReductionFlags(NoNaN);
 
   // All ops in the reduction inherit fast-math-flags from the recurrence
   // descriptor.
   IRBuilderBase::FastMathFlagGuard FMFGuard(B);
   B.setFastMathFlags(Desc.getFastMathFlags());
 
   switch (RecKind) {
   case RD::RK_FloatAdd:
-    return createSimpleTargetReduction(B, TTI, Instruction::FAdd, Src, Flags);
+    return createSimpleTargetReduction(B, Instruction::FAdd, Src,
+                                       UseReductionIntrinsic, Flags);
   case RD::RK_FloatMult:
-    return createSimpleTargetReduction(B, TTI, Instruction::FMul, Src, Flags);
+    return createSimpleTargetReduction(B, Instruction::FMul, Src,
+                                       UseReductionIntrinsic, Flags);
   case RD::RK_IntegerAdd:
-    return createSimpleTargetReduction(B, TTI, Instruction::Add, Src, Flags);
+    return createSimpleTargetReduction(B, Instruction::Add, Src,
+                                       UseReductionIntrinsic, Flags);
   case RD::RK_IntegerMult:
-    return createSimpleTargetReduction(B, TTI, Instruction::Mul, Src, Flags);
+    return createSimpleTargetReduction(B, Instruction::Mul, Src,
+                                       UseReductionIntrinsic, Flags);
   case RD::RK_IntegerAnd:
-    return createSimpleTargetReduction(B, TTI, Instruction::And, Src, Flags);
+    return createSimpleTargetReduction(B, Instruction::And, Src,
+                                       UseReductionIntrinsic, Flags);
   case RD::RK_IntegerOr:
-    return createSimpleTargetReduction(B, TTI, Instruction::Or, Src, Flags);
+    return createSimpleTargetReduction(B, Instruction::Or, Src,
+                                       UseReductionIntrinsic, Flags);
   case RD::RK_IntegerXor:
-    return createSimpleTargetReduction(B, TTI, Instruction::Xor, Src, Flags);
+    return createSimpleTargetReduction(B, Instruction::Xor, Src,
+                                       UseReductionIntrinsic, Flags);
   case RD::RK_IntegerMinMax: {
-    RD::MinMaxRecurrenceKind MMKind = Desc.getMinMaxRecurrenceKind();
-    Flags.IsMaxOp = (MMKind == RD::MRK_SIntMax || MMKind == RD::MRK_UIntMax);
-    Flags.IsSigned = (MMKind == RD::MRK_SIntMax || MMKind == RD::MRK_SIntMin);
-    return createSimpleTargetReduction(B, TTI, Instruction::ICmp, Src, Flags);
+    return createSimpleTargetReduction(B, Instruction::ICmp, Src,
+                                       UseReductionIntrinsic, Flags);
   }
   case RD::RK_FloatMinMax: {
-    Flags.IsMaxOp = Desc.getMinMaxRecurrenceKind() == RD::MRK_FloatMax;
-    return createSimpleTargetReduction(B, TTI, Instruction::FCmp, Src, Flags);
+    return createSimpleTargetReduction(B, Instruction::FCmp, Src,
+                                       UseReductionIntrinsic, Flags);
   }
   default:
     llvm_unreachable("Unhandled RecKind");
   }
 }
 
 void llvm::propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue) {
   auto *VecOp = dyn_cast<Instruction>(I);

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

for (unsigned Part = 0; Part < UF; ++Part) {
       // If the target can create a predicated operator for the reduction at no
       // extra cost in the loop (for example a predicated vadd), it can be
       // cheaper for the select to remain in the loop than be sunk out of it,
       // and so use the select value for the phi instead of the old
       // LoopExitValue.
       RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[Phi];
       if (TTI->preferInloopReductionSelect(
               RdxDesc.getRecurrenceBinOp(RdxDesc.getRecurrenceKind()),
-              Phi->getType(), TargetTransformInfo::ReductionFlags())) {
+              Phi->getType(),
+              RdxDesc.getReductionFlags(Legal->hasFunNoNaNAttr()))) {
         auto *VecRdxPhi = cast<PHINode>(getOrCreateVectorValue(Phi, Part));
         VecRdxPhi->setIncomingValueForBlock(
             LI->getLoopFor(LoopVectorBody)->getLoopLatch(), Sel);
       }
     }
   }
 
   // If the vector reduction can be performed in a smaller type, we truncate
else
       ReducedPartRdx = createMinMaxOp(Builder, MinMaxKind, ReducedPartRdx,
                                       RdxPart);
   }
 
   // Create the reduction after the loop. Note that inloop reductions create the
   // target reduction in the loop using a Reduction recipe.
   if (VF > 1 && !IsInLoopReductionPhi) {
     bool NoNaN = Legal->hasFunNoNaNAttr();
-    ReducedPartRdx =
-        createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx, NoNaN);
+    bool UseReductionIntrinsic = TTI->useReductionIntrinsic(
+        RdxDesc.getRecurrenceBinOp(RdxDesc.getRecurrenceKind()),
+        ReducedPartRdx->getType(),
+        RdxDesc.getReductionFlags(NoNaN));
+    ReducedPartRdx = createTargetReduction(Builder, RdxDesc, ReducedPartRdx,
+                                           UseReductionIntrinsic, NoNaN);
     // If the reduction can be performed in a smaller type, we need to extend
     // the reduction to the wider type before we branch to the original loop.
     if (Phi->getType() != RdxDesc.getRecurrenceType())
       ReducedPartRdx =
         RdxDesc.isSigned()
         ? Builder.CreateSExt(ReducedPartRdx, Phi->getType())
         : Builder.CreateZExt(ReducedPartRdx, Phi->getType());
   }
for (auto &Reduction : Legal->getReductionVars()) {
     // We don't collect reductions that are type promoted (yet).
     if (RdxDesc.getRecurrenceType() != Phi->getType())
       continue;
 
     // If the target would prefer this reduction to happen "in-loop", then we
     // want to record it as such.
     unsigned Opcode = RdxDesc.getRecurrenceBinOp(RdxDesc.getRecurrenceKind());
     if (!PreferInLoopReductions &&
-        !TTI.preferInLoopReduction(Opcode, Phi->getType(),
-                                   TargetTransformInfo::ReductionFlags()))
+        !TTI.preferInLoopReduction(
+            Opcode, Phi->getType(),
+            RdxDesc.getReductionFlags(Legal->hasFunNoNaNAttr())))
       continue;
 
     // Check that we can correctly put the reductions into the loop, by
     // finding the chain of operations that leads from the loop exit value back
     // to the phi.
     SmallVector<Instruction *, 4> ReductionOperations =
         RdxDesc.getReductionOpChain(Phi, TheLoop);
     bool InLoop = !ReductionOperations.empty();
for (Instruction *R : ReductionOperations) {
         assert(WidenRecipe->getVPRecipeID() == VPRecipeBase::VPWidenSC &&
                "Expected to replace a VPWidenSC");
         FirstOpId = 0;
       }
       unsigned VecOpId =
           R->getOperand(FirstOpId) == Chain ? FirstOpId + 1 : FirstOpId;
       VPValue *VecOp = Plan->getVPValue(R->getOperand(VecOpId));
 
+      bool UseReductionIntrinsic = TTI->useReductionIntrinsic(
+          RdxDesc.getRecurrenceBinOp(Kind), RdxDesc.getRecurrenceType(),
+          RdxDesc.getReductionFlags(Legal->hasFunNoNaNAttr()));
       VPReductionRecipe *RedRecipe = new VPReductionRecipe(
-          &RdxDesc, R, ChainOp, VecOp, Legal->hasFunNoNaNAttr(), TTI);
+          &RdxDesc, R, ChainOp, VecOp, Legal->hasFunNoNaNAttr(),
+          UseReductionIntrinsic);
       WidenRecipe->getParent()->insert(RedRecipe, WidenRecipe->getIterator());
       WidenRecipe->removeFromParent();
 
       if (Kind == RecurrenceDescriptor::RK_IntegerMinMax ||
           Kind == RecurrenceDescriptor::RK_FloatMinMax) {
         VPRecipeBase *CompareRecipe =
             RecipeBuilder.getRecipe(cast<Instruction>(R->getOperand(0)));
         CompareRecipe->removeFromParent();
void VPInterleaveRecipe::execute(VPTransformState &State) {
   State.ILV->vectorizeInterleaveGroup(IG, State, getAddr(), getMask());
 }
 
 void VPReductionRecipe::execute(VPTransformState &State) {
   assert(!State.Instance && "Reduction being replicated.");
   for (unsigned Part = 0; Part < State.UF; ++Part) {
     unsigned Kind = RdxDesc->getRecurrenceKind();
     Value *NewVecOp = State.get(VecOp, Part);
-    Value *NewRed =
-        createTargetReduction(State.Builder, TTI, *RdxDesc, NewVecOp, NoNaN);
+    Value *NewRed = createTargetReduction(State.Builder, *RdxDesc, NewVecOp,
+                                          UseReductionIntrinsic, NoNaN);
     Value *PrevInChain = State.get(ChainOp, Part);
     Value *NextInChain;
     if (Kind == RecurrenceDescriptor::RK_IntegerMinMax ||
         Kind == RecurrenceDescriptor::RK_FloatMinMax) {
       NextInChain =
           createMinMaxOp(State.Builder, RdxDesc->getMinMaxRecurrenceKind(),
                          NewRed, PrevInChain);
     } else {

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Value *emitReduction(Value *VectorizedValue, IRBuilder<> &Builder,
     assert(VectorizedValue && "Need to have a vectorized tree node");
     assert(isPowerOf2_32(ReduxWidth) &&
            "We only handle power-of-two reductions for now");
 
     if (!IsPairwiseReduction) {
       // FIXME: The builder should use an FMF guard. It should not be hard-coded
       //        to 'fast'.
       assert(Builder.getFastMathFlags().isFast() && "Expected 'fast' FMF");
-      return createSimpleTargetReduction(
-          Builder, TTI, ReductionData.getOpcode(), VectorizedValue,
-          ReductionData.getFlags(), ReductionOps.back());
+      TTI::ReductionFlags Flags = ReductionData.getFlags();
+      bool UseReductionIntrinsic = TTI->useReductionIntrinsic(
+          ReductionData.getOpcode(), VectorizedValue->getType(), Flags);
+      return createSimpleTargetReduction(Builder, ReductionData.getOpcode(),
+                                         VectorizedValue, UseReductionIntrinsic,
+                                         Flags, ReductionOps.back());
     }
 
     Value *TmpVec = VectorizedValue;
     for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) {
       auto LeftMask = createRdxShuffleMask(ReduxWidth, i, true, true);
       auto RightMask = createRdxShuffleMask(ReduxWidth, i, true, false);
 
       Value *LeftShuf = Builder.CreateShuffleVector(

llvm/lib/Transforms/Vectorize/VPlan.h

class VPReductionRecipe : public VPRecipeBase {
   /// The original instruction being converted to a reduction.
   Instruction *I;
   /// The VPValue of the vector value to be reduced.
   VPValue *VecOp;
   /// The VPValue of the scalar Chain being accumulated.
   VPValue *ChainOp;
   /// Fast math flags to use for the resulting reduction operation.
   bool NoNaN;
-  /// Pointer to the TTI, needed to create the target reduction
-  const TargetTransformInfo *TTI;
+  /// Flag for whether to use reduction intrinsics vs shuffle expansions.
+  bool UseReductionIntrinsic;
 
 public:
   VPReductionRecipe(RecurrenceDescriptor *R, Instruction *I, VPValue *ChainOp,
-                    VPValue *VecOp, bool NoNaN, const TargetTransformInfo *TTI)
+                    VPValue *VecOp, bool NoNaN, bool UseReductionIntrinsic)
       : VPRecipeBase(VPReductionSC), RdxDesc(R), I(I), VecOp(VecOp),
-        ChainOp(ChainOp), NoNaN(NoNaN), TTI(TTI) {}
+        ChainOp(ChainOp), NoNaN(NoNaN),
+        UseReductionIntrinsic(UseReductionIntrinsic) {}
 
   ~VPReductionRecipe() override = default;
 
   /// Method to support type inquiry through isa, cast, and dyn_cast.
   static inline bool classof(const VPRecipeBase *V) {
     return V->getVPRecipeID() == VPRecipeBase::VPReductionSC;
   }