diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1316,7 +1316,7 @@ /// RdxDesc. This is true if the -enable-strict-reductions flag is passed, /// the IsOrdered flag of RdxDesc is set and we do not allow reordering /// of FP operations. - bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) { + bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) const { return !Hints->allowReordering() && RdxDesc.isOrdered(); } @@ -1439,18 +1439,18 @@ /// Return the vectorization cost for the given instruction \p I and vector /// width \p VF. - InstructionCost getWideningCost(Instruction *I, ElementCount VF) { + InstructionCost getWideningCost(Instruction *I, ElementCount VF) const { assert(VF.isVector() && "Expected VF >=2"); std::pair InstOnVF = std::make_pair(I, VF); - assert(WideningDecisions.find(InstOnVF) != WideningDecisions.end() && - "The cost is not calculated"); - return WideningDecisions[InstOnVF].second; + auto Itr = WideningDecisions.find(InstOnVF); + assert(Itr != WideningDecisions.end() && "The cost is not calculated"); + return Itr->second.second; } /// Return True if instruction \p I is an optimizable truncate whose operand /// is an induction variable. Such a truncate will be removed by adding a new /// induction variable with the destination type. - bool isOptimizableIVTruncate(Instruction *I, ElementCount VF) { + bool isOptimizableIVTruncate(Instruction *I, ElementCount VF) const { // If the instruction is not a truncate, return false. auto *Trunc = dyn_cast(I); if (!Trunc) @@ -1679,25 +1679,26 @@ using InstructionVFPair = std::pair; VectorizationCostTy expectedCost(ElementCount VF, - SmallVectorImpl *Invalid = nullptr); + SmallVectorImpl *Invalid = nullptr) const; /// Returns the execution time cost of an instruction for a given vector /// width. Vector width of one means scalar. - VectorizationCostTy getInstructionCost(Instruction *I, ElementCount VF); + VectorizationCostTy getInstructionCost(Instruction *I, ElementCount VF) const; /// The cost-computation logic from getInstructionCost which provides /// the vector type as an output parameter. InstructionCost getInstructionCost(Instruction *I, ElementCount VF, - Type *&VectorTy); + Type *&VectorTy) const; /// Return the cost of instructions in an inloop reduction pattern, if I is /// part of that pattern. Optional getReductionPatternCost(Instruction *I, ElementCount VF, Type *VectorTy, - TTI::TargetCostKind CostKind); + TTI::TargetCostKind CostKind) const; /// Calculate vectorization cost of memory instruction \p I. - InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF); + InstructionCost getMemoryInstructionCost(Instruction *I, + ElementCount VF) const; /// The cost computation for scalarized memory instruction. InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF); @@ -6955,7 +6956,7 @@ LoopVectorizationCostModel::VectorizationCostTy LoopVectorizationCostModel::expectedCost( - ElementCount VF, SmallVectorImpl *Invalid) { + ElementCount VF, SmallVectorImpl *Invalid) const { VectorizationCostTy Cost; // For each block. @@ -7210,7 +7211,8 @@ } Optional LoopVectorizationCostModel::getReductionPatternCost( - Instruction *I, ElementCount VF, Type *Ty, TTI::TargetCostKind CostKind) { + Instruction *I, ElementCount VF, Type *Ty, + TTI::TargetCostKind CostKind) const { using namespace llvm::PatternMatch; // Early exit for no inloop reductions if (InLoopReductionChains.empty() || VF.isScalar() || !isa(Ty)) @@ -7244,16 +7246,21 @@ // Test if the found instruction is a reduction, and if not return an invalid // cost specifying the parent to use the original cost modelling. - if (!InLoopReductionImmediateChains.count(RetI)) + auto LastChainIter = InLoopReductionImmediateChains.find(RetI); + if (LastChainIter == InLoopReductionImmediateChains.end()) return None; // Find the reduction this chain is a part of and calculate the basic cost of // the reduction on its own. - Instruction *LastChain = InLoopReductionImmediateChains[RetI]; - Instruction *ReductionPhi = LastChain; - while (!isa(ReductionPhi)) - ReductionPhi = InLoopReductionImmediateChains[ReductionPhi]; - + Instruction *LastChain = LastChainIter->second; + auto ReductionPhiIter = LastChainIter; + while (!isa(ReductionPhiIter->second)) { + ReductionPhiIter = + InLoopReductionImmediateChains.find(ReductionPhiIter->second); + assert(ReductionPhiIter != InLoopReductionImmediateChains.end() && + "Failed to find reduction phi node for the chain"); + } + Instruction *ReductionPhi = ReductionPhiIter->second; const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[cast(ReductionPhi)]; @@ -7362,7 +7369,7 @@ InstructionCost LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I, - ElementCount VF) { + ElementCount VF) const { // Calculate scalar cost only. Vectorization cost should be ready at this // moment. if (VF.isScalar()) { @@ -7379,14 +7386,16 @@ LoopVectorizationCostModel::VectorizationCostTy LoopVectorizationCostModel::getInstructionCost(Instruction *I, - ElementCount VF) { + ElementCount VF) const { // If we know that this instruction will remain uniform, check the cost of // the scalar version. if (isUniformAfterVectorization(I, VF)) VF = ElementCount::getFixed(1); if (VF.isVector() && isProfitableToScalarize(I, VF)) - return VectorizationCostTy(InstsToScalarize[VF][I], false); + return VectorizationCostTy( + const_cast(this)->InstsToScalarize[VF][I], + false); // Forced scalars do not have any scalarization overhead. auto ForcedScalar = ForcedScalars.find(VF); @@ -7611,10 +7620,14 @@ InstructionCost LoopVectorizationCostModel::getInstructionCost(Instruction *I, ElementCount VF, - Type *&VectorTy) { + Type *&VectorTy) const { Type *RetTy = I->getType(); - if (canTruncateToMinimalBitwidth(I, VF)) - RetTy = IntegerType::get(RetTy->getContext(), MinBWs[I]); + if (canTruncateToMinimalBitwidth(I, VF)) { + auto MinBWIter = MinBWs.find(I); + assert(MinBWIter != MinBWs.end() && + "Can't find minimum bit width for the instruction"); + RetTy = IntegerType::get(RetTy->getContext(), MinBWIter->second); + } auto SE = PSE.getSE(); TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; @@ -7820,8 +7833,12 @@ case Instruction::FCmp: { Type *ValTy = I->getOperand(0)->getType(); Instruction *Op0AsInstruction = dyn_cast(I->getOperand(0)); - if (canTruncateToMinimalBitwidth(Op0AsInstruction, VF)) - ValTy = IntegerType::get(ValTy->getContext(), MinBWs[Op0AsInstruction]); + if (canTruncateToMinimalBitwidth(Op0AsInstruction, VF)) { + auto MinBWIter = MinBWs.find(Op0AsInstruction); + assert(MinBWIter != MinBWs.end() && + "Can't find minimum bit width for the instruction"); + ValTy = IntegerType::get(ValTy->getContext(), MinBWIter->second); + } VectorTy = ToVectorTy(ValTy, VF); return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, nullptr, CmpInst::BAD_ICMP_PREDICATE, CostKind, I);