Index: lib/CodeGen/MachineLICM.cpp =================================================================== --- lib/CodeGen/MachineLICM.cpp +++ lib/CodeGen/MachineLICM.cpp @@ -267,9 +267,20 @@ /// this does not count live through (livein but not used) registers. void InitRegPressure(MachineBasicBlock *BB); + /// calcRegisterCost - Calculate the additional register pressure that the + /// registers used in MI cause. + /// + /// If 'ConsiderSeen' is true, updates 'RegSeen' and uses the information to + /// figure out which usages are live-ins. + /// FIXME: Figure out a way to consider 'RegSeen' from all code paths. + DenseMap calcRegisterCost(const MachineInstr *MI, + bool ConsiderSeen, + bool ConsiderUnseenAsDef); + /// UpdateRegPressure - Update estimate of register pressure after the /// specified instruction. - void UpdateRegPressure(const MachineInstr *MI); + void UpdateRegPressure(const MachineInstr *MI, + bool ConsiderUnseenAsDef = false); /// ExtractHoistableLoad - Unfold a load from the given machineinstr if /// the load itself could be hoisted. Return the unfolded and hoistable @@ -871,41 +882,30 @@ InitRegPressure(*BB->pred_begin()); } - for (MachineBasicBlock::iterator MII = BB->begin(), E = BB->end(); - MII != E; ++MII) { - MachineInstr *MI = &*MII; - for (unsigned i = 0, e = MI->getDesc().getNumOperands(); i != e; ++i) { - const MachineOperand &MO = MI->getOperand(i); - if (!MO.isReg() || MO.isImplicit()) - continue; - unsigned Reg = MO.getReg(); - if (!TargetRegisterInfo::isVirtualRegister(Reg)) - continue; - - bool isNew = RegSeen.insert(Reg).second; - unsigned RCId, RCCost; - getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost); - if (MO.isDef()) - RegPressure[RCId] += RCCost; - else { - bool isKill = isOperandKill(MO, MRI); - if (isNew && !isKill) - // Haven't seen this, it must be a livein. - RegPressure[RCId] += RCCost; - else if (!isNew && isKill) - RegPressure[RCId] -= RCCost; - } - } - } + for (const MachineInstr &MI : *BB) + UpdateRegPressure(&MI, /*ConsiderUnseenAsDef=*/true); } /// UpdateRegPressure - Update estimate of register pressure after the /// specified instruction. -void MachineLICM::UpdateRegPressure(const MachineInstr *MI) { - if (MI->isImplicitDef()) - return; +void MachineLICM::UpdateRegPressure(const MachineInstr *MI, + bool ConsiderUnseenAsDef) { + auto Cost = calcRegisterCost(MI, /*ConsiderSeen=*/true, ConsiderUnseenAsDef); + for (const auto &ClassAndCost : Cost) { + unsigned Class = ClassAndCost.first; + if (static_cast(RegPressure[Class]) < -ClassAndCost.second) + RegPressure[Class] = 0; + else + RegPressure[Class] += ClassAndCost.second; + } +} - SmallVector Defs; +DenseMap +MachineLICM::calcRegisterCost(const MachineInstr *MI, bool ConsiderSeen, + bool ConsiderUnseenAsDef) { + DenseMap Cost; + if (MI->isImplicitDef()) + return Cost; for (unsigned i = 0, e = MI->getDesc().getNumOperands(); i != e; ++i) { const MachineOperand &MO = MI->getOperand(i); if (!MO.isReg() || MO.isImplicit()) @@ -914,27 +914,25 @@ if (!TargetRegisterInfo::isVirtualRegister(Reg)) continue; - bool isNew = RegSeen.insert(Reg).second; + // FIXME: It seems bad to use RegSeen only for some of these calculations. + bool isNew = ConsiderSeen ? RegSeen.insert(Reg).second : false; + unsigned RCId, RCCost; + getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost); + int PriorCost = 0; + if (Cost.find(RCId) != Cost.end()) + PriorCost = Cost[RCId]; if (MO.isDef()) - Defs.push_back(Reg); - else if (!isNew && isOperandKill(MO, MRI)) { - unsigned RCId, RCCost; - getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost); - if (RCCost > RegPressure[RCId]) - RegPressure[RCId] = 0; - else - RegPressure[RCId] -= RCCost; + Cost[RCId] = PriorCost + RCCost; + else { + bool isKill = isOperandKill(MO, MRI); + if (isNew && !isKill && ConsiderUnseenAsDef) + // Haven't seen this, it must be a livein. + Cost[RCId] = PriorCost + RCCost; + else if (!isNew && isKill) + Cost[RCId] = PriorCost - RCCost; } } - - unsigned Idx = 0; - while (!Defs.empty()) { - unsigned Reg = Defs.pop_back_val(); - unsigned RCId, RCCost; - getRegisterClassIDAndCost(MI, Reg, Idx, RCId, RCCost); - RegPressure[RCId] += RCCost; - ++Idx; - } + return Cost; } /// isLoadFromGOTOrConstantPool - Return true if this machine instruction @@ -1152,46 +1150,15 @@ /// current block and update their register pressures to reflect the effect /// of hoisting MI from the current block to the preheader. void MachineLICM::UpdateBackTraceRegPressure(const MachineInstr *MI) { - if (MI->isImplicitDef()) - return; - // First compute the 'cost' of the instruction, i.e. its contribution // to register pressure. - DenseMap Cost; - for (unsigned i = 0, e = MI->getDesc().getNumOperands(); i != e; ++i) { - const MachineOperand &MO = MI->getOperand(i); - if (!MO.isReg() || MO.isImplicit()) - continue; - unsigned Reg = MO.getReg(); - if (!TargetRegisterInfo::isVirtualRegister(Reg)) - continue; - - unsigned RCId, RCCost; - getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost); - if (MO.isDef()) { - DenseMap::iterator CI = Cost.find(RCId); - if (CI != Cost.end()) - CI->second += RCCost; - else - Cost.insert(std::make_pair(RCId, RCCost)); - } else if (isOperandKill(MO, MRI)) { - DenseMap::iterator CI = Cost.find(RCId); - if (CI != Cost.end()) - CI->second -= RCCost; - else - Cost.insert(std::make_pair(RCId, -RCCost)); - } - } + auto Cost = calcRegisterCost(MI, /*ConsiderSeen=*/false, + /*ConsiderUnseenAsDef=*/false); // Update register pressure of blocks from loop header to current block. - for (unsigned i = 0, e = BackTrace.size(); i != e; ++i) { - SmallVectorImpl &RP = BackTrace[i]; - for (DenseMap::iterator CI = Cost.begin(), CE = Cost.end(); - CI != CE; ++CI) { - unsigned RCId = CI->first; - RP[RCId] += CI->second; - } - } + for (auto &RP : BackTrace) + for (const auto &ClassAndCost : Cost) + RP[ClassAndCost.first] += ClassAndCost.second; } /// IsProfitableToHoist - Return true if it is potentially profitable to hoist @@ -1226,15 +1193,8 @@ if (TII->isTriviallyReMaterializable(&MI, AA)) return true; - // Estimate register pressure to determine whether to LICM the instruction. - // In low register pressure situation, we can be more aggressive about - // hoisting. Also, favors hoisting long latency instructions even in - // moderately high pressure situation. - // Cheap instructions will only be hoisted if they don't increase register - // pressure at all. // FIXME: If there are long latency loop-invariant instructions inside the // loop at this point, why didn't the optimizer's LICM hoist them? - DenseMap Cost; for (unsigned i = 0, e = MI.getDesc().getNumOperands(); i != e; ++i) { const MachineOperand &MO = MI.getOperand(i); if (!MO.isReg() || MO.isImplicit()) @@ -1242,24 +1202,22 @@ unsigned Reg = MO.getReg(); if (!TargetRegisterInfo::isVirtualRegister(Reg)) continue; - - unsigned RCId, RCCost; - getRegisterClassIDAndCost(&MI, Reg, i, RCId, RCCost); - if (MO.isDef()) { - if (HasHighOperandLatency(MI, i, Reg)) { - DEBUG(dbgs() << "Hoist High Latency: " << MI); - ++NumHighLatency; - return true; - } - Cost[RCId] += RCCost; - } else if (isOperandKill(MO, MRI)) { - // Is a virtual register use is a kill, hoisting it out of the loop - // may actually reduce register pressure or be register pressure - // neutral. - Cost[RCId] -= RCCost; + if (MO.isDef() && HasHighOperandLatency(MI, i, Reg)) { + DEBUG(dbgs() << "Hoist High Latency: " << MI); + ++NumHighLatency; + return true; } } + // Estimate register pressure to determine whether to LICM the instruction. + // In low register pressure situation, we can be more aggressive about + // hoisting. Also, favors hoisting long latency instructions even in + // moderately high pressure situation. + // Cheap instructions will only be hoisted if they don't increase register + // pressure at all. + auto Cost = calcRegisterCost(&MI, /*ConsiderSeen=*/false, + /*ConsiderUnseenAsDef=*/false); + // Visit BBs from header to current BB, if hoisting this doesn't cause // high register pressure, then it's safe to proceed. if (!CanCauseHighRegPressure(Cost, CheapInstr)) {