Index: llvm/lib/Transforms/Utils/SimplifyCFG.cpp =================================================================== --- llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -335,8 +335,8 @@ /// which is assumed to be safe to speculate. TCC_Free means cheap, /// TCC_Basic means less cheap, and TCC_Expensive means prohibitively /// expensive. -static unsigned computeSpeculationCost(const User *I, - const TargetTransformInfo &TTI) { +static InstructionCost computeSpeculationCost(const User *I, + const TargetTransformInfo &TTI) { assert(isSafeToSpeculativelyExecute(I) && "Instruction is not safe to speculatively execute!"); return TTI.getUserCost(I, TargetTransformInfo::TCK_SizeAndLatency); @@ -361,7 +361,8 @@ /// CostRemaining, false is returned and CostRemaining is undefined. static bool dominatesMergePoint(Value *V, BasicBlock *BB, SmallPtrSetImpl &AggressiveInsts, - int &BudgetRemaining, + InstructionCost &Cost, + InstructionCost Budget, const TargetTransformInfo &TTI, unsigned Depth = 0) { // It is possible to hit a zero-cost cycle (phi/gep instructions for example), @@ -404,7 +405,7 @@ if (!isSafeToSpeculativelyExecute(I)) return false; - BudgetRemaining -= computeSpeculationCost(I, TTI); + Cost += computeSpeculationCost(I, TTI); // Allow exactly one instruction to be speculated regardless of its cost // (as long as it is safe to do so). @@ -412,14 +413,15 @@ // or other expensive operation. The speculation of an expensive instruction // is expected to be undone in CodeGenPrepare if the speculation has not // enabled further IR optimizations. - if (BudgetRemaining < 0 && - (!SpeculateOneExpensiveInst || !AggressiveInsts.empty() || Depth > 0)) + if (Cost > Budget && + (!SpeculateOneExpensiveInst || !AggressiveInsts.empty() || Depth > 0 || + !Cost.isValid())) return false; // Okay, we can only really hoist these out if their operands do // not take us over the cost threshold. for (User::op_iterator i = I->op_begin(), e = I->op_end(); i != e; ++i) - if (!dominatesMergePoint(*i, BB, AggressiveInsts, BudgetRemaining, TTI, + if (!dominatesMergePoint(*i, BB, AggressiveInsts, Cost, Budget, TTI, Depth + 1)) return false; // Okay, it's safe to do this! Remember this instruction. @@ -2113,7 +2115,7 @@ static bool validateAndCostRequiredSelects(BasicBlock *BB, BasicBlock *ThenBB, BasicBlock *EndBB, unsigned &SpeculatedInstructions, - int &BudgetRemaining, + InstructionCost &Cost, const TargetTransformInfo &TTI) { TargetTransformInfo::TargetCostKind CostKind = BB->getParent()->hasMinSize() @@ -2130,9 +2132,8 @@ if (ThenV == OrigV) continue; - BudgetRemaining -= - TTI.getCmpSelInstrCost(Instruction::Select, PN.getType(), nullptr, - CmpInst::BAD_ICMP_PREDICATE, CostKind); + Cost += TTI.getCmpSelInstrCost(Instruction::Select, PN.getType(), nullptr, + CmpInst::BAD_ICMP_PREDICATE, CostKind); // Don't convert to selects if we could remove undefined behavior instead. if (passingValueIsAlwaysUndefined(OrigV, &PN) || @@ -2148,9 +2149,9 @@ if ((ThenCE && !isSafeToSpeculativelyExecute(ThenCE)) || (OrigCE && !isSafeToSpeculativelyExecute(OrigCE))) return false; - unsigned OrigCost = OrigCE ? computeSpeculationCost(OrigCE, TTI) : 0; - unsigned ThenCost = ThenCE ? computeSpeculationCost(ThenCE, TTI) : 0; - unsigned MaxCost = + InstructionCost OrigCost = OrigCE ? computeSpeculationCost(OrigCE, TTI) : 0; + InstructionCost ThenCost = ThenCE ? computeSpeculationCost(ThenCE, TTI) : 0; + InstructionCost MaxCost = 2 * PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic; if (OrigCost + ThenCost > MaxCost) return false; @@ -2213,8 +2214,8 @@ BasicBlock *BB = BI->getParent(); BasicBlock *EndBB = ThenBB->getTerminator()->getSuccessor(0); - int BudgetRemaining = - PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic; + InstructionCost Budget = + PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic; // If ThenBB is actually on the false edge of the conditional branch, remember // to swap the select operands later. @@ -2303,10 +2304,11 @@ // Check that we can insert the selects and that it's not too expensive to do // so. bool Convert = SpeculatedStore != nullptr; + InstructionCost Cost = 0; Convert |= validateAndCostRequiredSelects(BB, ThenBB, EndBB, SpeculatedInstructions, - BudgetRemaining, TTI); - if (!Convert || BudgetRemaining < 0) + Cost, TTI); + if (!Convert || Cost > Budget) return false; // If we get here, we can hoist the instruction and if-convert. @@ -2560,7 +2562,8 @@ // instructions. While we are at it, keep track of the instructions // that need to be moved to the dominating block. SmallPtrSet AggressiveInsts; - int BudgetRemaining = + InstructionCost Cost = 0; + InstructionCost Budget = TwoEntryPHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic; bool Changed = false; @@ -2574,9 +2577,9 @@ } if (!dominatesMergePoint(PN->getIncomingValue(0), BB, AggressiveInsts, - BudgetRemaining, TTI) || + Cost, Budget, TTI) || !dominatesMergePoint(PN->getIncomingValue(1), BB, AggressiveInsts, - BudgetRemaining, TTI)) + Cost, Budget, TTI)) return Changed; } @@ -3051,7 +3054,7 @@ // transformation. if (TTI) { Type *Ty = BI->getCondition()->getType(); - unsigned Cost = TTI->getArithmeticInstrCost(Opc, Ty, CostKind); + InstructionCost Cost = TTI->getArithmeticInstrCost(Opc, Ty, CostKind); if (InvertPredCond && (!PBI->getCondition()->hasOneUse() || !isa(PBI->getCondition()))) Cost += TTI->getArithmeticInstrCost(Instruction::Xor, Ty, CostKind); @@ -3185,7 +3188,8 @@ // Heuristic: if the block can be if-converted/phi-folded and the // instructions inside are all cheap (arithmetic/GEPs), it's worthwhile to // thread this store. - int BudgetRemaining = + InstructionCost Cost = 0; + InstructionCost Budget = PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic; for (auto &I : BB->instructionsWithoutDebug()) { // Consider terminator instruction to be free. @@ -3201,11 +3205,11 @@ return false; // Not in white-list - not worthwhile folding. // And finally, if this is a non-free instruction that we are okay // speculating, ensure that we consider the speculation budget. - BudgetRemaining -= TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency); - if (BudgetRemaining < 0) + Cost += TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency); + if (Cost > Budget) return false; // Eagerly refuse to fold as soon as we're out of budget. } - assert(BudgetRemaining >= 0 && + assert(Cost <= Budget && "When we run out of budget we will eagerly return from within the " "per-instruction loop."); return true;