diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1695,9 +1695,9 @@ /// Return the cost of instructions in an inloop reduction pattern, if I is /// part of that pattern. - InstructionCost getReductionPatternCost(Instruction *I, ElementCount VF, - Type *VectorTy, - TTI::TargetCostKind CostKind); + Optional + getReductionPatternCost(Instruction *I, ElementCount VF, Type *VectorTy, + TTI::TargetCostKind CostKind); /// Calculate vectorization cost of memory instruction \p I. InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF); @@ -7125,11 +7125,11 @@ return Cost; } -InstructionCost LoopVectorizationCostModel::getReductionPatternCost( +Optional LoopVectorizationCostModel::getReductionPatternCost( Instruction *I, ElementCount VF, Type *Ty, TTI::TargetCostKind CostKind) { // Early exit for no inloop reductions if (InLoopReductionChains.empty() || VF.isScalar() || !isa(Ty)) - return InstructionCost::getInvalid(); + return None; auto *VectorTy = cast(Ty); // We are looking for a pattern of, and finding the minimal acceptable cost: @@ -7148,20 +7148,20 @@ if ((RetI->getOpcode() == Instruction::SExt || RetI->getOpcode() == Instruction::ZExt)) { if (!RetI->hasOneUser()) - return InstructionCost::getInvalid(); + return None; RetI = RetI->user_back(); } if (RetI->getOpcode() == Instruction::Mul && RetI->user_back()->getOpcode() == Instruction::Add) { if (!RetI->hasOneUser()) - return InstructionCost::getInvalid(); + return None; RetI = RetI->user_back(); } // Test if the found instruction is a reduction, and if not return an invalid // cost specifying the parent to use the original cost modelling. if (!InLoopReductionImmediateChains.count(RetI)) - return InstructionCost::getInvalid(); + return None; // Find the reduction this chain is a part of and calculate the basic cost of // the reduction on its own. @@ -7195,7 +7195,7 @@ TTI.getCastInstrCost(RedOp->getOpcode(), VectorTy, ExtType, TTI::CastContextHint::None, CostKind, RedOp); if (RedCost.isValid() && RedCost < BaseCost + ExtCost) - return I == RetI ? *RedCost.getValue() : 0; + return I == RetI ? RedCost : 0; } else if (RedOp && RedOp->getOpcode() == Instruction::Mul) { Instruction *Mul = RedOp; Instruction *Op0 = dyn_cast(Mul->getOperand(0)); @@ -7218,7 +7218,7 @@ CostKind); if (RedCost.isValid() && RedCost < ExtCost * 2 + MulCost + BaseCost) - return I == RetI ? *RedCost.getValue() : 0; + return I == RetI ? RedCost : 0; } else { InstructionCost MulCost = TTI.getArithmeticInstrCost(Mul->getOpcode(), VectorTy, CostKind); @@ -7228,11 +7228,11 @@ CostKind); if (RedCost.isValid() && RedCost < MulCost + BaseCost) - return I == RetI ? *RedCost.getValue() : 0; + return I == RetI ? RedCost : 0; } } - return I == RetI ? BaseCost : InstructionCost::getInvalid(); + return I == RetI ? Optional(BaseCost) : None; } InstructionCost @@ -7637,10 +7637,8 @@ return 0; // Detect reduction patterns - InstructionCost RedCost; - if ((RedCost = getReductionPatternCost(I, VF, VectorTy, CostKind)) - .isValid()) - return RedCost; + if (auto RedCost = getReductionPatternCost(I, VF, VectorTy, CostKind)) + return *RedCost; // Certain instructions can be cheaper to vectorize if they have a constant // second vector operand. One example of this are shifts on x86. @@ -7781,10 +7779,8 @@ } // Detect reduction patterns - InstructionCost RedCost; - if ((RedCost = getReductionPatternCost(I, VF, VectorTy, CostKind)) - .isValid()) - return RedCost; + if (auto RedCost = getReductionPatternCost(I, VF, VectorTy, CostKind)) + return *RedCost; Type *SrcScalarTy = I->getOperand(0)->getType(); Type *SrcVecTy =