diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -773,6 +773,9 @@ /// Return true if target doesn't mind addresses in vectors. bool prefersVectorizedAddressing() const; + /// Returns true if target is ok with scalarizing vector code. + bool isOkToScalarize() const; + /// Return the cost of the scaling factor used in the addressing /// mode represented by AM for this target, for a load/store /// of the specified type. @@ -1758,6 +1761,7 @@ virtual bool hasDivRemOp(Type *DataType, bool IsSigned) = 0; virtual bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) = 0; virtual bool prefersVectorizedAddressing() = 0; + virtual bool isOkToScalarize() = 0; virtual InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, @@ -2226,6 +2230,7 @@ bool prefersVectorizedAddressing() override { return Impl.prefersVectorizedAddressing(); } + bool isOkToScalarize() override { return Impl.isOkToScalarize(); } InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -297,6 +297,8 @@ bool prefersVectorizedAddressing() const { return true; } + bool isOkToScalarize() const { return true; } + InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h --- a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h +++ b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h @@ -44,11 +44,6 @@ /// This class provides both the logic to recursively visit instructions and /// combine them. class LLVM_LIBRARY_VISIBILITY InstCombiner { - /// Only used to call target specific intrinsic combining. - /// It must **NOT** be used for any other purpose, as InstCombine is a - /// target-independent canonicalization transform. - TargetTransformInfo &TTI; - public: /// Maximum size of array considered when transforming. uint64_t MaxArraySizeForCombine = 0; @@ -59,6 +54,11 @@ BuilderTy &Builder; protected: + /// Only used to call target specific intrinsic combining. + /// It must **NOT** be used for any other purpose, as InstCombine is a + /// target-independent canonicalization transform. + TargetTransformInfo &TTI; + /// A worklist of the instructions that need to be simplified. InstructionWorklist &Worklist; @@ -90,7 +90,7 @@ DominatorTree &DT, OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, const DataLayout &DL, LoopInfo *LI) - : TTI(TTI), Builder(Builder), Worklist(Worklist), + : Builder(Builder), TTI(TTI), Worklist(Worklist), MinimizeSize(MinimizeSize), AA(AA), AC(AC), TLI(TLI), DT(DT), DL(DL), SQ(DL, &TLI, &DT, &AC), ORE(ORE), BFI(BFI), PSI(PSI), LI(LI) {} diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -489,6 +489,10 @@ return TTIImpl->prefersVectorizedAddressing(); } +bool TargetTransformInfo::isOkToScalarize() const { + return TTIImpl->isOkToScalarize(); +} + InstructionCost TargetTransformInfo::getScalingFactorCost( Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) const { diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -20,6 +20,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/InstructionSimplify.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constant.h" @@ -55,7 +56,9 @@ /// some operations may be cheap to scalarize. /// /// FIXME: It's possible to create more instructions than previously existed. -static bool cheapToScalarize(Value *V, Value *EI) { +static bool cheapToScalarize(Value *V, Value *EI, TargetTransformInfo &TTI) { + if (!TTI.isOkToScalarize()) + return false; ConstantInt *CEI = dyn_cast(EI); // If we can pick a scalar constant value out of a vector, that is free. @@ -83,12 +86,12 @@ Value *V0, *V1; if (match(V, m_OneUse(m_BinOp(m_Value(V0), m_Value(V1))))) - if (cheapToScalarize(V0, EI) || cheapToScalarize(V1, EI)) + if (cheapToScalarize(V0, EI, TTI) || cheapToScalarize(V1, EI, TTI)) return true; CmpInst::Predicate UnusedPred; if (match(V, m_OneUse(m_Cmp(UnusedPred, m_Value(V0), m_Value(V1))))) - if (cheapToScalarize(V0, EI) || cheapToScalarize(V1, EI)) + if (cheapToScalarize(V0, EI, TTI) || cheapToScalarize(V1, EI, TTI)) return true; return false; @@ -126,7 +129,7 @@ // otherwise return nullptr. if (!PHIUser->hasOneUse() || !(PHIUser->user_back() == PN) || !(isa(PHIUser)) || - !cheapToScalarize(PHIUser, EI.getIndexOperand())) + !cheapToScalarize(PHIUser, EI.getIndexOperand(), TTI)) return nullptr; // Create a scalar PHI node that will replace the vector PHI node @@ -461,7 +464,7 @@ // TODO come up with a n-ary matcher that subsumes both unary and // binary matchers. UnaryOperator *UO; - if (match(SrcVec, m_UnOp(UO)) && cheapToScalarize(SrcVec, Index)) { + if (match(SrcVec, m_UnOp(UO)) && cheapToScalarize(SrcVec, Index, TTI)) { // extelt (unop X), Index --> unop (extelt X, Index) Value *X = UO->getOperand(0); Value *E = Builder.CreateExtractElement(X, Index); @@ -469,7 +472,7 @@ } BinaryOperator *BO; - if (match(SrcVec, m_BinOp(BO)) && cheapToScalarize(SrcVec, Index)) { + if (match(SrcVec, m_BinOp(BO)) && cheapToScalarize(SrcVec, Index, TTI)) { // extelt (binop X, Y), Index --> binop (extelt X, Index), (extelt Y, Index) Value *X = BO->getOperand(0), *Y = BO->getOperand(1); Value *E0 = Builder.CreateExtractElement(X, Index); @@ -480,7 +483,7 @@ Value *X, *Y; CmpInst::Predicate Pred; if (match(SrcVec, m_Cmp(Pred, m_Value(X), m_Value(Y))) && - cheapToScalarize(SrcVec, Index)) { + cheapToScalarize(SrcVec, Index, TTI)) { // extelt (cmp X, Y), Index --> cmp (extelt X, Index), (extelt Y, Index) Value *E0 = Builder.CreateExtractElement(X, Index); Value *E1 = Builder.CreateExtractElement(Y, Index);