diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h --- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h +++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h @@ -24,6 +24,7 @@ #include "llvm/IR/IRBuilder.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/InstructionCost.h" namespace llvm { extern cl::opt SCEVCheapExpansionBudget; @@ -237,15 +238,16 @@ return true; // by always claiming to be high-cost. SmallVector Worklist; SmallPtrSet Processed; - int BudgetRemaining = Budget * TargetTransformInfo::TCC_Basic; + InstructionCost Cost = 0; + unsigned ScaledBudget = Budget * TargetTransformInfo::TCC_Basic; Worklist.emplace_back(-1, -1, Expr); while (!Worklist.empty()) { const SCEVOperand WorkItem = Worklist.pop_back_val(); - if (isHighCostExpansionHelper(WorkItem, L, *At, BudgetRemaining, - *TTI, Processed, Worklist)) + if (isHighCostExpansionHelper(WorkItem, L, *At, Cost, ScaledBudget, *TTI, + Processed, Worklist)) return true; } - assert(BudgetRemaining >= 0 && "Should have returned from inner loop."); + assert(Cost <= ScaledBudget && "Should have returned from inner loop."); return false; } @@ -399,11 +401,12 @@ Value *expandCodeForImpl(const SCEV *SH, Type *Ty, Instruction *I, bool Root); /// Recursive helper function for isHighCostExpansion. - bool isHighCostExpansionHelper( - const SCEVOperand &WorkItem, Loop *L, const Instruction &At, - int &BudgetRemaining, const TargetTransformInfo &TTI, - SmallPtrSetImpl &Processed, - SmallVectorImpl &Worklist); + bool isHighCostExpansionHelper(const SCEVOperand &WorkItem, Loop *L, + const Instruction &At, InstructionCost &Cost, + unsigned Budget, + const TargetTransformInfo &TTI, + SmallPtrSetImpl &Processed, + SmallVectorImpl &Worklist); /// Insert the specified binary operator, doing a small amount of work to /// avoid inserting an obviously redundant operation, and hoisting to an diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -2179,13 +2179,13 @@ return None; } -template static int costAndCollectOperands( +template static InstructionCost costAndCollectOperands( const SCEVOperand &WorkItem, const TargetTransformInfo &TTI, TargetTransformInfo::TargetCostKind CostKind, SmallVectorImpl &Worklist) { const T *S = cast(WorkItem.S); - int Cost = 0; + InstructionCost Cost = 0; // Object to help map SCEV operands to expanded IR instructions. struct OperationIndices { OperationIndices(unsigned Opc, size_t min, size_t max) : @@ -2200,7 +2200,7 @@ // we know what the generated user(s) will be. SmallVector Operations; - auto CastCost = [&](unsigned Opcode) { + auto CastCost = [&](unsigned Opcode) -> InstructionCost { Operations.emplace_back(Opcode, 0, 0); return TTI.getCastInstrCost(Opcode, S->getType(), S->getOperand(0)->getType(), @@ -2208,14 +2208,15 @@ }; auto ArithCost = [&](unsigned Opcode, unsigned NumRequired, - unsigned MinIdx = 0, unsigned MaxIdx = 1) { + unsigned MinIdx = 0, + unsigned MaxIdx = 1) -> InstructionCost { Operations.emplace_back(Opcode, MinIdx, MaxIdx); return NumRequired * TTI.getArithmeticInstrCost(Opcode, S->getType(), CostKind); }; - auto CmpSelCost = [&](unsigned Opcode, unsigned NumRequired, - unsigned MinIdx, unsigned MaxIdx) { + auto CmpSelCost = [&](unsigned Opcode, unsigned NumRequired, unsigned MinIdx, + unsigned MaxIdx) -> InstructionCost { Operations.emplace_back(Opcode, MinIdx, MaxIdx); Type *OpType = S->getOperand(0)->getType(); return NumRequired * TTI.getCmpSelInstrCost( @@ -2286,10 +2287,11 @@ // Much like with normal add expr, the polynominal will require // one less addition than the number of it's terms. - int AddCost = ArithCost(Instruction::Add, NumTerms - 1, - /*MinIdx*/1, /*MaxIdx*/1); + InstructionCost AddCost = ArithCost(Instruction::Add, NumTerms - 1, + /*MinIdx*/ 1, /*MaxIdx*/ 1); // Here, *each* one of those will require a multiplication. - int MulCost = ArithCost(Instruction::Mul, NumNonZeroDegreeNonOneTerms); + InstructionCost MulCost = + ArithCost(Instruction::Mul, NumNonZeroDegreeNonOneTerms); Cost = AddCost + MulCost; // What is the degree of this polynominal? @@ -2320,10 +2322,10 @@ bool SCEVExpander::isHighCostExpansionHelper( const SCEVOperand &WorkItem, Loop *L, const Instruction &At, - int &BudgetRemaining, const TargetTransformInfo &TTI, + InstructionCost &Cost, unsigned Budget, const TargetTransformInfo &TTI, SmallPtrSetImpl &Processed, SmallVectorImpl &Worklist) { - if (BudgetRemaining < 0) + if (Cost > Budget) return true; // Already run out of budget, give up. const SCEV *S = WorkItem.S; @@ -2353,17 +2355,16 @@ return 0; const APInt &Imm = cast(S)->getAPInt(); Type *Ty = S->getType(); - BudgetRemaining -= TTI.getIntImmCostInst( + Cost += TTI.getIntImmCostInst( WorkItem.ParentOpcode, WorkItem.OperandIdx, Imm, Ty, CostKind); - return BudgetRemaining < 0; + return Cost > Budget; } case scTruncate: case scPtrToInt: case scZeroExtend: case scSignExtend: { - int Cost = + Cost += costAndCollectOperands(WorkItem, TTI, CostKind, Worklist); - BudgetRemaining -= Cost; return false; // Will answer upon next entry into this function. } case scUDivExpr: { @@ -2379,10 +2380,8 @@ SE.getAddExpr(S, SE.getConstant(S->getType(), 1)), &At, L)) return false; // Consider it to be free. - int Cost = + Cost += costAndCollectOperands(WorkItem, TTI, CostKind, Worklist); - // Need to count the cost of this UDiv. - BudgetRemaining -= Cost; return false; // Will answer upon next entry into this function. } case scAddExpr: @@ -2395,17 +2394,16 @@ "Nary expr should have more than 1 operand."); // The simple nary expr will require one less op (or pair of ops) // than the number of it's terms. - int Cost = + Cost += costAndCollectOperands(WorkItem, TTI, CostKind, Worklist); - BudgetRemaining -= Cost; - return BudgetRemaining < 0; + return Cost > Budget; } case scAddRecExpr: { assert(cast(S)->getNumOperands() >= 2 && "Polynomial should be at least linear"); - BudgetRemaining -= costAndCollectOperands( + Cost += costAndCollectOperands( WorkItem, TTI, CostKind, Worklist); - return BudgetRemaining < 0; + return Cost > Budget; } } llvm_unreachable("Unknown SCEV kind!");