diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp --- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp +++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp @@ -1021,11 +1021,12 @@ do { LocalChange = false; + FunctionAnalysisManager &FAM = + AM.getResult(C, CG).getManager(); + for (LazyCallGraph::Node &N : C) { Function &OldF = N.getFunction(); - FunctionAnalysisManager &FAM = - AM.getResult(C, CG).getManager(); // FIXME: This lambda must only be used with this function. We should // skip the lambda and just get the AA results directly. auto AARGetter = [&](Function &F) -> AAResults & { @@ -1048,6 +1049,11 @@ C.getOuterRefSCC().replaceNodeFunction(N, *NewF); FAM.clear(OldF, OldF.getName()); OldF.eraseFromParent(); + + for (auto *U : NewF->users()) { + auto *UserF = cast(U)->getParent()->getParent(); + FAM.invalidate(*UserF, PreservedAnalyses::none()); + } } Changed |= LocalChange; @@ -1056,7 +1062,10 @@ if (!Changed) return PreservedAnalyses::all(); - return PreservedAnalyses::none(); + PreservedAnalyses PA; + PA.preserve(); + PA.preserveSet>(); + return PA; } namespace { diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp --- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp +++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp @@ -240,7 +240,8 @@ /// Deduce readonly/readnone attributes for the SCC. template -static bool addReadAttrs(const SCCNodeSet &SCCNodes, AARGetterT &&AARGetter) { +static void addReadAttrs(const SCCNodeSet &SCCNodes, AARGetterT &&AARGetter, + SmallSetVector &Changed) { // Check if any of the functions in the SCC read or write memory. If they // write memory then they can't be marked readnone or readonly. bool ReadsMemory = false; @@ -255,7 +256,7 @@ switch (checkFunctionMemoryAccess(*F, F->hasExactDefinition(), AAR, SCCNodes)) { case MAK_MayWrite: - return false; + return; case MAK_ReadOnly: ReadsMemory = true; break; @@ -271,11 +272,10 @@ // If the SCC contains both functions that read and functions that write, then // we cannot add readonly attributes. if (ReadsMemory && WritesMemory) - return false; + return; // Success! Functions in this SCC do not access memory, or only read memory. // Give them the appropriate attribute. - bool MadeChange = false; for (Function *F : SCCNodes) { if (F->doesNotAccessMemory()) @@ -289,7 +289,7 @@ if (F->doesNotReadMemory() && WritesMemory) continue; - MadeChange = true; + Changed.insert(F); // Clear out any existing attributes. AttrBuilder AttrsToRemove; @@ -318,8 +318,6 @@ else ++NumReadNone; } - - return MadeChange; } namespace { @@ -581,9 +579,8 @@ } /// Deduce returned attributes for the SCC. -static bool addArgumentReturnedAttrs(const SCCNodeSet &SCCNodes) { - bool Changed = false; - +static void addArgumentReturnedAttrs(const SCCNodeSet &SCCNodes, + SmallSetVector &Changed) { // Check each function in turn, determining if an argument is always returned. for (Function *F : SCCNodes) { // We can infer and propagate function attributes only when we know that the @@ -623,11 +620,9 @@ auto *A = cast(RetArg); A->addAttr(Attribute::Returned); ++NumReturned; - Changed = true; + Changed.insert(F); } } - - return Changed; } /// If a callsite has arguments that are also arguments to the parent function, @@ -693,9 +688,8 @@ } /// Deduce nocapture attributes for the SCC. -static bool addArgumentAttrs(const SCCNodeSet &SCCNodes) { - bool Changed = false; - +static void addArgumentAttrs(const SCCNodeSet &SCCNodes, + SmallSetVector &Changed) { ArgumentGraph AG; // Check each function in turn, determining which pointer arguments are not @@ -707,7 +701,8 @@ if (!F->hasExactDefinition()) continue; - Changed |= addArgumentAttrsFromCallsites(*F); + if (addArgumentAttrsFromCallsites(*F)) + Changed.insert(F); // Functions that are readonly (or readnone) and nounwind and don't return // a value can't capture arguments. Don't analyze them. @@ -718,7 +713,7 @@ if (A->getType()->isPointerTy() && !A->hasNoCaptureAttr()) { A->addAttr(Attribute::NoCapture); ++NumNoCapture; - Changed = true; + Changed.insert(F); } } continue; @@ -737,7 +732,7 @@ // If it's trivially not captured, mark it nocapture now. A->addAttr(Attribute::NoCapture); ++NumNoCapture; - Changed = true; + Changed.insert(F); } else { // If it's not trivially captured and not trivially not captured, // then it must be calling into another function in our SCC. Save @@ -761,7 +756,8 @@ Self.insert(&*A); Attribute::AttrKind R = determinePointerReadAttrs(&*A, Self); if (R != Attribute::None) - Changed = addReadAttr(A, R); + if (addReadAttr(A, R)) + Changed.insert(F); } } } @@ -785,7 +781,7 @@ Argument *A = ArgumentSCC[0]->Definition; A->addAttr(Attribute::NoCapture); ++NumNoCapture; - Changed = true; + Changed.insert(A->getParent()); } continue; } @@ -827,7 +823,7 @@ Argument *A = ArgumentSCC[i]->Definition; A->addAttr(Attribute::NoCapture); ++NumNoCapture; - Changed = true; + Changed.insert(A->getParent()); } // We also want to compute readonly/readnone. With a small number of false @@ -858,12 +854,11 @@ if (ReadAttr != Attribute::None) { for (unsigned i = 0, e = ArgumentSCC.size(); i != e; ++i) { Argument *A = ArgumentSCC[i]->Definition; - Changed = addReadAttr(A, ReadAttr); + if (addReadAttr(A, ReadAttr)) + Changed.insert(A->getParent()); } } } - - return Changed; } /// Tests whether a function is "malloc-like". @@ -934,7 +929,8 @@ } /// Deduce noalias attributes for the SCC. -static bool addNoAliasAttrs(const SCCNodeSet &SCCNodes) { +static void addNoAliasAttrs(const SCCNodeSet &SCCNodes, + SmallSetVector &Changed) { // Check each function in turn, determining which functions return noalias // pointers. for (Function *F : SCCNodes) { @@ -946,7 +942,7 @@ // definition we'll get at link time is *exactly* the definition we see now. // For more details, see GlobalValue::mayBeDerefined. if (!F->hasExactDefinition()) - return false; + return; // We annotate noalias return values, which are only applicable to // pointer types. @@ -954,10 +950,9 @@ continue; if (!isFunctionMallocLike(F, SCCNodes)) - return false; + return; } - bool MadeChange = false; for (Function *F : SCCNodes) { if (F->returnDoesNotAlias() || !F->getReturnType()->isPointerTy()) @@ -965,10 +960,8 @@ F->setReturnDoesNotAlias(); ++NumNoAlias; - MadeChange = true; + Changed.insert(F); } - - return MadeChange; } /// Tests whether this function is known to not return null. @@ -1044,13 +1037,12 @@ } /// Deduce nonnull attributes for the SCC. -static bool addNonNullAttrs(const SCCNodeSet &SCCNodes) { +static void addNonNullAttrs(const SCCNodeSet &SCCNodes, + SmallSetVector &Changed) { // Speculative that all functions in the SCC return only nonnull // pointers. We may refute this as we analyze functions. bool SCCReturnsNonNull = true; - bool MadeChange = false; - // Check each function in turn, determining which functions return nonnull // pointers. for (Function *F : SCCNodes) { @@ -1063,7 +1055,7 @@ // definition we'll get at link time is *exactly* the definition we see now. // For more details, see GlobalValue::mayBeDerefined. if (!F->hasExactDefinition()) - return false; + return; // We annotate nonnull return values, which are only applicable to // pointer types. @@ -1079,7 +1071,7 @@ << " as nonnull\n"); F->addAttribute(AttributeList::ReturnIndex, Attribute::NonNull); ++NumNonNullReturn; - MadeChange = true; + Changed.insert(F); } continue; } @@ -1098,11 +1090,9 @@ LLVM_DEBUG(dbgs() << "SCC marking " << F->getName() << " as nonnull\n"); F->addAttribute(AttributeList::ReturnIndex, Attribute::NonNull); ++NumNonNullReturn; - MadeChange = true; + Changed.insert(F); } } - - return MadeChange; } namespace { @@ -1155,12 +1145,13 @@ InferenceDescriptors.push_back(AttrInference); } - bool run(const SCCNodeSet &SCCNodes); + void run(const SCCNodeSet &SCCNodes, SmallSetVector &Changed); }; /// Perform all the requested attribute inference actions according to the /// attribute predicates stored before. -bool AttributeInferer::run(const SCCNodeSet &SCCNodes) { +void AttributeInferer::run(const SCCNodeSet &SCCNodes, + SmallSetVector &Changed) { SmallVector InferInSCC = InferenceDescriptors; // Go through all the functions in SCC and check corresponding attribute // assumptions for each of them. Attributes that are invalid for this SCC @@ -1169,7 +1160,7 @@ // No attributes whose assumptions are still valid - done. if (InferInSCC.empty()) - return false; + return; // Check if our attributes ever need scanning/can be scanned. llvm::erase_if(InferInSCC, [F](const InferenceDescriptor &ID) { @@ -1212,9 +1203,8 @@ } if (InferInSCC.empty()) - return false; + return; - bool Changed = false; for (Function *F : SCCNodes) // At this point InferInSCC contains only functions that were either: // - explicitly skipped from scan/inference, or @@ -1223,10 +1213,9 @@ for (auto &ID : InferInSCC) { if (ID.SkipFunction(*F)) continue; - Changed = true; + Changed.insert(F); ID.SetAttribute(*F); } - return Changed; } struct SCCNodesResult { @@ -1282,7 +1271,8 @@ /// Attempt to remove convergent function attribute when possible. /// /// Returns true if any changes to function attributes were made. -static bool inferConvergent(const SCCNodeSet &SCCNodes) { +static void inferConvergent(const SCCNodeSet &SCCNodes, + SmallSetVector &Changed) { AttributeInferer AI; // Request to remove the convergent attribute from all functions in the SCC @@ -1305,7 +1295,7 @@ }, /* RequiresExactDefinition= */ false}); // Perform all the requested attribute inference actions. - return AI.run(SCCNodes); + AI.run(SCCNodes, Changed); } /// Infer attributes from all functions in the SCC by scanning every @@ -1314,7 +1304,9 @@ /// - addition of NoUnwind attribute /// /// Returns true if any changes to function attributes were made. -static bool inferAttrsFromFunctionBodies(const SCCNodeSet &SCCNodes) { +static void +inferAttrsFromFunctionBodies(const SCCNodeSet &SCCNodes, + SmallSetVector &Changed) { AttributeInferer AI; if (!DisableNoUnwindInference) @@ -1363,19 +1355,20 @@ /* RequiresExactDefinition= */ true}); // Perform all the requested attribute inference actions. - return AI.run(SCCNodes); + AI.run(SCCNodes, Changed); } -static bool addNoRecurseAttrs(const SCCNodeSet &SCCNodes) { +static void addNoRecurseAttrs(const SCCNodeSet &SCCNodes, + SmallSetVector &Changed) { // Try and identify functions that do not recurse. // If the SCC contains multiple nodes we know for sure there is recursion. if (SCCNodes.size() != 1) - return false; + return; Function *F = *SCCNodes.begin(); if (!F || !F->hasExactDefinition() || F->doesNotRecurse()) - return false; + return; // If all of the calls in F are identifiable and are to norecurse functions, F // is norecurse. This check also detects self-recursion as F is not currently @@ -1386,7 +1379,7 @@ Function *Callee = CB->getCalledFunction(); if (!Callee || Callee == F || !Callee->doesNotRecurse()) // Function calls a potentially recursive function. - return false; + return; } // Every call was to a non-recursive function other than this function, and @@ -1394,7 +1387,7 @@ // recurse. F->setDoesNotRecurse(); ++NumNoRecurse; - return true; + Changed.insert(F); } static bool instructionDoesNotReturn(Instruction &I) { @@ -1412,9 +1405,8 @@ } // Set the noreturn function attribute if possible. -static bool addNoReturnAttrs(const SCCNodeSet &SCCNodes) { - bool Changed = false; - +static void addNoReturnAttrs(const SCCNodeSet &SCCNodes, + SmallSetVector &Changed) { for (Function *F : SCCNodes) { if (!F || !F->hasExactDefinition() || F->hasFnAttribute(Attribute::Naked) || F->doesNotReturn()) @@ -1424,11 +1416,9 @@ // FIXME: this doesn't handle recursion or unreachable blocks. if (none_of(*F, basicBlockCanReturn)) { F->setDoesNotReturn(); - Changed = true; + Changed.insert(F); } } - - return Changed; } static bool functionWillReturn(const Function &F) { @@ -1455,19 +1445,16 @@ } // Set the willreturn function attribute if possible. -static bool addWillReturn(const SCCNodeSet &SCCNodes) { - bool Changed = false; - +static void addWillReturn(const SCCNodeSet &SCCNodes, + SmallSetVector &Changed) { for (Function *F : SCCNodes) { if (!F || F->willReturn() || !functionWillReturn(*F)) continue; F->setWillReturn(); NumWillReturn++; - Changed = true; + Changed.insert(F); } - - return Changed; } // Return true if this is an atomic which has an ordering stronger than @@ -1526,7 +1513,8 @@ } // Infer the nosync attribute. -static bool addNoSyncAttr(const SCCNodeSet &SCCNodes) { +static void addNoSyncAttr(const SCCNodeSet &SCCNodes, + SmallSetVector &Changed) { AttributeInferer AI; AI.registerAttrInference(AttributeInferer::InferenceDescriptor{ Attribute::NoSync, @@ -1543,7 +1531,7 @@ ++NumNoSync; }, /* RequiresExactDefinition= */ true}); - return AI.run(SCCNodes); + AI.run(SCCNodes, Changed); } static SCCNodesResult createSCCNodeSet(ArrayRef Functions) { @@ -1576,32 +1564,33 @@ } template -static bool deriveAttrsInPostOrder(ArrayRef Functions, - AARGetterT &&AARGetter) { +static SmallSetVector +deriveAttrsInPostOrder(ArrayRef Functions, AARGetterT &&AARGetter) { SCCNodesResult Nodes = createSCCNodeSet(Functions); - bool Changed = false; // Bail if the SCC only contains optnone functions. if (Nodes.SCCNodes.empty()) - return Changed; + return {}; - Changed |= addArgumentReturnedAttrs(Nodes.SCCNodes); - Changed |= addReadAttrs(Nodes.SCCNodes, AARGetter); - Changed |= addArgumentAttrs(Nodes.SCCNodes); - Changed |= inferConvergent(Nodes.SCCNodes); - Changed |= addNoReturnAttrs(Nodes.SCCNodes); - Changed |= addWillReturn(Nodes.SCCNodes); + SmallSetVector Changed; + + addArgumentReturnedAttrs(Nodes.SCCNodes, Changed); + addReadAttrs(Nodes.SCCNodes, AARGetter, Changed); + addArgumentAttrs(Nodes.SCCNodes, Changed); + inferConvergent(Nodes.SCCNodes, Changed); + addNoReturnAttrs(Nodes.SCCNodes, Changed); + addWillReturn(Nodes.SCCNodes, Changed); // If we have no external nodes participating in the SCC, we can deduce some // more precise attributes as well. if (!Nodes.HasUnknownCall) { - Changed |= addNoAliasAttrs(Nodes.SCCNodes); - Changed |= addNonNullAttrs(Nodes.SCCNodes); - Changed |= inferAttrsFromFunctionBodies(Nodes.SCCNodes); - Changed |= addNoRecurseAttrs(Nodes.SCCNodes); + addNoAliasAttrs(Nodes.SCCNodes, Changed); + addNonNullAttrs(Nodes.SCCNodes, Changed); + inferAttrsFromFunctionBodies(Nodes.SCCNodes, Changed); + addNoRecurseAttrs(Nodes.SCCNodes, Changed); } - Changed |= addNoSyncAttr(Nodes.SCCNodes); + addNoSyncAttr(Nodes.SCCNodes, Changed); // Finally, infer the maximal set of attributes from the ones we've inferred // above. This is handling the cases where one attribute on a signature @@ -1609,7 +1598,8 @@ // the later is missing (or simply less sophisticated). for (Function *F : Nodes.SCCNodes) if (F) - Changed |= inferAttributesFromOthers(*F); + if (inferAttributesFromOthers(*F)) + Changed.insert(F); return Changed; } @@ -1632,14 +1622,18 @@ Functions.push_back(&N.getFunction()); } - if (deriveAttrsInPostOrder(Functions, AARGetter)) { - // We have not changed the call graph or removed/added functions. - PreservedAnalyses PA; - PA.preserve(); - return PA; + auto ChangedFunctions = deriveAttrsInPostOrder(Functions, AARGetter); + if (ChangedFunctions.empty()) { + return PreservedAnalyses::all(); } - return PreservedAnalyses::all(); + for (Function *Changed : ChangedFunctions) + FAM.invalidate(*Changed, PreservedAnalyses::none()); + // We have not changed the call graph or removed/added functions. + PreservedAnalyses PA; + PA.preserve(); + PA.preserveSet>(); + return PA; } namespace { @@ -1684,7 +1678,7 @@ Functions.push_back(I->getFunction()); } - return deriveAttrsInPostOrder(Functions, AARGetter); + return !deriveAttrsInPostOrder(Functions, AARGetter).empty(); } bool PostOrderFunctionAttrsLegacyPass::runOnSCC(CallGraphSCC &SCC) { diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -951,6 +951,10 @@ UR.InlinedInternalEdges.insert({&N, OldC}); } InlinedCallees.clear(); + + // Invalidate analyses for this function now so that we don't have to + // invalidate analyses for all functions in this SCC later. + FAM.invalidate(F, PreservedAnalyses::none()); } // Now that we've finished inlining all of the calls across this SCC, delete @@ -990,10 +994,12 @@ if (!Changed) return PreservedAnalyses::all(); + PreservedAnalyses PA; // Even if we change the IR, we update the core CGSCC data structures and so // can preserve the proxy to the function analysis manager. - PreservedAnalyses PA; PA.preserve(); + // We have already invalidated all analyses on modified functions. + PA.preserveSet>(); return PA; }