Index: lib/Transforms/Scalar/LoopStrengthReduce.cpp =================================================================== --- lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -1062,6 +1062,57 @@ } }; +/// Helper class to count probability as a fixed point instead of float. +class FixedPoint64 { +private: + uint64_t Value; + FixedPoint64(uint64_t N) : Value(N) {} + FixedPoint64(uint32_t Hi, uint32_t Lo) { + Value = (((uint64_t)Hi) << 32) | Lo; + } + uint32_t getLo() const { + return (uint32_t)Value; + } + uint32_t getHi() const { + return (uint32_t)(Value >> 32); + } +public: + FixedPoint64() : Value(0) {} + FixedPoint64(uint32_t N) { + Value = FixedPoint64(N, 0).Value; + } + double getFloat() const { + return (float)Value / ((uint64_t)1 << 32); + } + FixedPoint64 operator+(const FixedPoint64 Add) const { + return FixedPoint64(Value + Add.Value); + } + FixedPoint64 operator/(const FixedPoint64 &Divider) const { + assert(Divider.Value && "Division by zero!"); + uint32_t Hi = (uint32_t)(Value / Divider.Value); + uint32_t Lo = (uint32_t)((Value << 32) / Divider.Value); + if (Divider.Value >> 32) + Lo += (uint32_t)((Value) / (Divider.Value >> 32)); + return FixedPoint64(Hi, Lo); + } + FixedPoint64 operator*(const FixedPoint64 Multiplier) const { + uint64_t Sum = (uint64_t)this->getHi() * Multiplier.getLo(); + Sum += (uint64_t)this->getLo() * Multiplier.getHi(); + Sum += ((uint64_t)this->getLo() * Multiplier.getLo()) >> 32; + uint32_t Hi = (uint32_t)(this->getHi() * Multiplier.getHi() + (Sum >> 32)); + return FixedPoint64(Hi, (uint32_t)Sum); + } + bool operator>(const FixedPoint64 Right) const { + return Value > Right.Value; + } + bool operator!=(const FixedPoint64 Right) const { + return Value != Right.Value; + } + bool operator==(const FixedPoint64 Right) const { + return Value == Right.Value; + } +}; + /// This class holds the state that LSR keeps for each use in IVUsers, as well /// as uses invented by LSR itself. It includes information about what kinds of /// things can be folded into the user, information about the user itself, and @@ -1137,7 +1188,7 @@ } bool HasFormulaWithSameRegs(const Formula &F) const; - float getNotSelectedProbability(const SCEV *Reg) const; + FixedPoint64 getNotSelectedProbability(const SCEV *Reg) const; bool InsertFormula(const Formula &F, const Loop &L); void DeleteFormula(Formula &F); void RecomputeRegs(size_t LUIdx, RegUseTracker &Reguses); @@ -1417,12 +1468,13 @@ } /// The function returns a probability of selecting formula without Reg. -float LSRUse::getNotSelectedProbability(const SCEV *Reg) const { - unsigned FNum = 0; +FixedPoint64 LSRUse::getNotSelectedProbability(const SCEV *Reg) const { + uint32_t FSize = Formulae.size(); + uint32_t FNum = FSize; for (const Formula &F : Formulae) if (F.referencesReg(Reg)) - FNum++; - return ((float)(Formulae.size() - FNum)) / Formulae.size(); + FNum--; + return FixedPoint64(FNum) / FixedPoint64(FSize); } /// If the given formula has not yet been inserted, add it to the list, and @@ -4363,17 +4415,17 @@ DEBUG(dbgs() << "The search space is too complex.\n"); // Map each register to probability of not selecting - DenseMap RegNumMap; + DenseMap RegNumMap; for (const SCEV *Reg : RegUses) { if (UniqRegs.count(Reg)) continue; - float PNotSel = 1; + FixedPoint64 PNotSel((uint32_t)1); for (const LSRUse &LU : Uses) { if (!LU.Regs.count(Reg)) continue; - float P = LU.getNotSelectedProbability(Reg); - if (P != 0.0) - PNotSel *= P; + FixedPoint64 P = LU.getNotSelectedProbability(Reg); + if (P != FixedPoint64((uint32_t)0)) + PNotSel = PNotSel * P; else UniqRegs.insert(Reg); } @@ -4391,27 +4443,28 @@ // This is temporary solution to test performance. Float should be // replaced with round independent type (based on integers) to avoid // different results for different target builds. - float FMinRegNum = LU.Formulae[0].getNumRegs(); - float FMinARegNum = LU.Formulae[0].getNumRegs(); + FixedPoint64 FMinRegNum((uint32_t)LU.Formulae[0].getNumRegs()); + FixedPoint64 FMinARegNum((uint32_t)LU.Formulae[0].getNumRegs()); size_t MinIdx = 0; for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { Formula &F = LU.Formulae[i]; - float FRegNum = 0; - float FARegNum = 0; + FixedPoint64 FRegNum; + FixedPoint64 FARegNum; for (const SCEV *BaseReg : F.BaseRegs) { if (UniqRegs.count(BaseReg)) continue; - FRegNum += RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg); + FRegNum = FRegNum + + RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg); if (isa(BaseReg)) - FARegNum += + FARegNum = FARegNum + RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg); } if (const SCEV *ScaledReg = F.ScaledReg) { if (!UniqRegs.count(ScaledReg)) { - FRegNum += + FRegNum = FRegNum + RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg); if (isa(ScaledReg)) - FARegNum += + FARegNum = FARegNum + RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg); } } @@ -4423,7 +4476,7 @@ } } DEBUG(dbgs() << " The formula "; LU.Formulae[MinIdx].print(dbgs()); - dbgs() << " with min reg num " << FMinRegNum << '\n'); + dbgs() << " with min reg num " << FMinRegNum.getFloat() << '\n'); if (MinIdx != 0) std::swap(LU.Formulae[MinIdx], LU.Formulae[0]); while (LU.Formulae.size() != 1) {