Index: llvm/lib/CodeGen/CodeGenPrepare.cpp =================================================================== --- llvm/lib/CodeGen/CodeGenPrepare.cpp +++ llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -304,7 +304,8 @@ const TargetTransformInfo *TTI = nullptr; const BasicBlockSectionsProfileReader *BBSectionsProfileReader = nullptr; const TargetLibraryInfo *TLInfo = nullptr; - const LoopInfo *LI = nullptr; + const LoopInfo *InitialLI = nullptr; + std::unique_ptr LI; std::unique_ptr BFI; std::unique_ptr BPI; ProfileSummaryInfo *PSI = nullptr; @@ -408,6 +409,12 @@ } } + void resetDT() { + DT.reset(); + LI.reset(); + InitialLI = nullptr; + } + // Get the DominatorTree, building if necessary. DominatorTree &getDT(Function &F) { if (!DT) @@ -415,6 +422,14 @@ return *DT; } + const LoopInfo &getLI(Function &F) { + if (InitialLI) + return *InitialLI; + if (!LI) + LI = std::make_unique(getDT(F)); + return *LI; + } + void removeAllAssertingVHReferences(Value *V); bool eliminateAssumptions(Function &F); bool eliminateFallThrough(Function &F); @@ -509,9 +524,11 @@ TRI = SubtargetInfo->getRegisterInfo(); TLInfo = &getAnalysis().getTLI(F); TTI = &getAnalysis().getTTI(F); - LI = &getAnalysis().getLoopInfo(); - BPI.reset(new BranchProbabilityInfo(F, *LI)); - BFI.reset(new BlockFrequencyInfo(F, *BPI, *LI)); + InitialLI = &getAnalysis().getLoopInfo(); + + + BPI.reset(new BranchProbabilityInfo(F, *InitialLI)); + BFI.reset(new BlockFrequencyInfo(F, *BPI, *InitialLI)); PSI = &getAnalysis().getPSI(); BBSectionsProfileReader = getAnalysisIfAvailable(); @@ -582,7 +599,7 @@ bool FuncIterated = false; while (MadeChange) { MadeChange = false; - DT.reset(); + resetDT(); for (BasicBlock &BB : llvm::make_early_inc_range(F)) { if (FuncIterated && !FreshBBs.contains(&BB)) @@ -608,7 +625,7 @@ FreshBBs.erase(&BB); if (ModifiedDTOnIteration == ModifyDT::ModifyBBDT) - DT.reset(); + resetDT(); } else { // For small/normal functions, we restart BB iteration if the dominator // tree of the Function was changed. @@ -845,7 +862,7 @@ /// blocks so we can split them the way we want them. bool CodeGenPrepare::eliminateMostlyEmptyBlocks(Function &F) { SmallPtrSet Preheaders; - SmallVector LoopList(LI->begin(), LI->end()); + SmallVector LoopList(getLI(F).begin(), getLI(F).end()); while (!LoopList.empty()) { Loop *L = LoopList.pop_back_val(); llvm::append_range(LoopList, *L); @@ -1473,18 +1490,20 @@ CmpInst *Cmp, Intrinsic::ID IID) { auto IsReplacableIVIncrement = [this, &Cmp](BinaryOperator *BO) { - if (!isIVIncrement(BO, LI)) + Function &F = *BO->getParent()->getParent(); + const LoopInfo &LI = getLI(F); + if (!isIVIncrement(BO, &LI)) return false; - const Loop *L = LI->getLoopFor(BO->getParent()); + const Loop *L = LI.getLoopFor(BO->getParent()); assert(L && "L should not be null after isIVIncrement()"); // Do not risk on moving increment into a child loop. - if (LI->getLoopFor(Cmp->getParent()) != L) + if (LI.getLoopFor(Cmp->getParent()) != L) return false; // Finally, we need to ensure that the insert point will dominate all // existing uses of the increment. - auto &DT = getDT(*BO->getParent()->getParent()); + auto &DT = getDT(F); if (DT.dominates(Cmp->getParent(), BO->getParent())) // If we're moving up the dom tree, all uses are trivially dominated. // (This is the common case for code produced by LSR.) @@ -3217,8 +3236,8 @@ const TargetLowering &TLI; const TargetRegisterInfo &TRI; const DataLayout &DL; - const LoopInfo &LI; const std::function getDTFn; + const std::function getLIFn; /// AccessTy/MemoryInst - This is the type for the access (e.g. double) and /// the memory instruction that we're computing this address for. @@ -3254,18 +3273,19 @@ AddressingModeMatcher( SmallVectorImpl &AMI, const TargetLowering &TLI, - const TargetRegisterInfo &TRI, const LoopInfo &LI, - const std::function getDTFn, Type *AT, - unsigned AS, Instruction *MI, ExtAddrMode &AM, - const SetOfInstrs &InsertedInsts, InstrToOrigTy &PromotedInsts, - TypePromotionTransaction &TPT, + const TargetRegisterInfo &TRI, + const std::function getDTFn, + const std::function getLIFn, Type *AT, unsigned AS, + Instruction *MI, ExtAddrMode &AM, const SetOfInstrs &InsertedInsts, + InstrToOrigTy &PromotedInsts, TypePromotionTransaction &TPT, std::pair, int64_t> &LargeOffsetGEP, bool OptSize, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) : AddrModeInsts(AMI), TLI(TLI), TRI(TRI), - DL(MI->getModule()->getDataLayout()), LI(LI), getDTFn(getDTFn), - AccessTy(AT), AddrSpace(AS), MemoryInst(MI), AddrMode(AM), - InsertedInsts(InsertedInsts), PromotedInsts(PromotedInsts), TPT(TPT), - LargeOffsetGEP(LargeOffsetGEP), OptSize(OptSize), PSI(PSI), BFI(BFI) { + DL(MI->getModule()->getDataLayout()), getDTFn(getDTFn), + getLIFn(getLIFn), AccessTy(AT), AddrSpace(AS), MemoryInst(MI), + AddrMode(AM), InsertedInsts(InsertedInsts), + PromotedInsts(PromotedInsts), TPT(TPT), LargeOffsetGEP(LargeOffsetGEP), + OptSize(OptSize), PSI(PSI), BFI(BFI) { IgnoreProfitability = false; } @@ -3280,18 +3300,19 @@ static ExtAddrMode Match(Value *V, Type *AccessTy, unsigned AS, Instruction *MemoryInst, SmallVectorImpl &AddrModeInsts, - const TargetLowering &TLI, const LoopInfo &LI, + const TargetLowering &TLI, const std::function getDTFn, + const std::function getLIFn, const TargetRegisterInfo &TRI, const SetOfInstrs &InsertedInsts, InstrToOrigTy &PromotedInsts, TypePromotionTransaction &TPT, std::pair, int64_t> &LargeOffsetGEP, bool OptSize, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) { ExtAddrMode Result; - bool Success = AddressingModeMatcher(AddrModeInsts, TLI, TRI, LI, getDTFn, - AccessTy, AS, MemoryInst, Result, - InsertedInsts, PromotedInsts, TPT, - LargeOffsetGEP, OptSize, PSI, BFI) + bool Success = AddressingModeMatcher(AddrModeInsts, TLI, TRI, getDTFn, + getLIFn, AccessTy, AS, MemoryInst, + Result, InsertedInsts, PromotedInsts, + TPT, LargeOffsetGEP, OptSize, PSI, BFI) .matchAddr(V, 0); (void)Success; assert(Success && "Couldn't select *anything*?"); @@ -4000,7 +4021,7 @@ Value *AddLHS = nullptr; if (isa(ScaleReg) && // not a constant expr. match(ScaleReg, m_Add(m_Value(AddLHS), m_ConstantInt(CI))) && - !isIVIncrement(ScaleReg, &LI) && CI->getValue().isSignedIntN(64)) { + !isIVIncrement(ScaleReg, &getLIFn()) && CI->getValue().isSignedIntN(64)) { TestAddrMode.InBounds = false; TestAddrMode.ScaledReg = AddLHS; TestAddrMode.BaseOffs += CI->getSExtValue() * TestAddrMode.Scale; @@ -4023,7 +4044,7 @@ auto *PN = dyn_cast(V); if (!PN) return std::nullopt; - auto IVInc = getIVIncrement(PN, &LI); + auto IVInc = getIVIncrement(PN, &getLIFn()); if (!IVInc) return std::nullopt; // TODO: The result of the intrinsics above is two-complement. However when @@ -4058,7 +4079,7 @@ // This transforms is (intentionally) the inverse of the one just above. // If they don't agree on the definition of an increment, we'd alternate // back and forth indefinitely. - assert(isIVIncrement(IVInc, &LI) && "implied by GetConstantStep"); + assert(isIVIncrement(IVInc, &getLIFn()) && "implied by GetConstantStep"); APInt Step = IVStep->second; APInt Offset = Step * AddrMode.Scale; if (Offset.isSignedIntN(64)) { @@ -5170,8 +5191,8 @@ 0); TypePromotionTransaction::ConstRestorationPt LastKnownGood = TPT.getRestorationPoint(); - AddressingModeMatcher Matcher(MatchedAddrModeInsts, TLI, TRI, LI, getDTFn, - AddressAccessTy, AS, UserI, Result, + AddressingModeMatcher Matcher(MatchedAddrModeInsts, TLI, TRI, getDTFn, + getLIFn, AddressAccessTy, AS, UserI, Result, InsertedInsts, PromotedInsts, TPT, LargeOffsetGEP, OptSize, PSI, BFI); Matcher.IgnoreProfitability = true; @@ -5283,10 +5304,14 @@ Function *F = MemoryInst->getParent()->getParent(); return this->getDT(*F); }; + auto getLIFn = [MemoryInst, this]() -> const LoopInfo & { + Function *F = MemoryInst->getParent()->getParent(); + return this->getLI(*F); + }; ExtAddrMode NewAddrMode = AddressingModeMatcher::Match( - V, AccessTy, AddrSpace, MemoryInst, AddrModeInsts, *TLI, *LI, getDTFn, - *TRI, InsertedInsts, PromotedInsts, TPT, LargeOffsetGEP, OptSize, PSI, - BFI.get()); + V, AccessTy, AddrSpace, MemoryInst, AddrModeInsts, *TLI, getDTFn, + getLIFn, *TRI, InsertedInsts, PromotedInsts, TPT, LargeOffsetGEP, + OptSize, PSI, BFI.get()); GetElementPtrInst *GEP = LargeOffsetGEP.first; if (GEP && !NewGEPBases.count(GEP)) { @@ -6902,7 +6927,7 @@ // transformation. We simply reset here rather than setting the ModifiedDT // flag to avoid restarting the function walk in runOnFunction for each // select optimized. - DT.reset(); + resetDT(); // Transform a sequence like this: // start: @@ -8064,9 +8089,10 @@ if (OptimizeNoopCopyExpression(CI, *TLI, *DL)) return true; + const LoopInfo &LI = getLI(*I->getParent()->getParent()); if ((isa(I) || isa(I) || isa(I)) && TLI->optimizeExtendOrTruncateConversion(I, - LI->getLoopFor(I->getParent()))) + LI.getLoopFor(I->getParent()))) return true; if (isa(I) || isa(I)) { @@ -8078,7 +8104,7 @@ return SinkCast(CI); } else { if (TLI->optimizeExtendOrTruncateConversion( - I, LI->getLoopFor(I->getParent()))) + I, LI.getLoopFor(I->getParent()))) return true; bool MadeChange = optimizeExt(I); @@ -8242,13 +8268,11 @@ // For huge function we tend to quickly go though the inner optmization // opportunities in the BB. So we go back to the BB head to re-optimize // each instruction instead of go back to the function head. - if (IsHugeFunc) { - DT.reset(); - getDT(*BB.getParent()); - break; - } else { + if (!IsHugeFunc) return true; - } + + resetDT(); + break; } } } while (ModifiedDT == ModifyDT::ModifyInstDT);