Index: lib/Transforms/Utils/InlineFunction.cpp =================================================================== --- lib/Transforms/Utils/InlineFunction.cpp +++ lib/Transforms/Utils/InlineFunction.cpp @@ -63,6 +63,90 @@ } namespace { +class InlineSite { + Function *CalledFunction; + Instruction *CallSiteInst; + CallSite::arg_iterator ArgBegin; + CallSite::arg_iterator ArgEnd; + unsigned ArgSize; + bool DoesNotThrow; + +#ifdef NDEBUG + void verify() const {} +#else + void verify() const; +#endif + + bool paramHasAttr(unsigned i, Attribute::AttrKind A) const { + assert(i < ArgSize && "out of bounds!"); + if (auto *II = dyn_cast(CallSiteInst)) + return II->paramHasAttr(i, A); + else + return cast(CallSiteInst)->paramHasAttr(i, A); + } + +public: + explicit InlineSite(CallSite CS) + : CalledFunction(CS.getCalledFunction()), + CallSiteInst(CS.getInstruction()), ArgBegin(CS.arg_begin()), + ArgEnd(CS.arg_end()), ArgSize(CS.arg_size()), + DoesNotThrow(CS.doesNotThrow()) { + verify(); + } + + Function *getCalledFunction() const { return CalledFunction; } + + Function *getCaller() const; + + Instruction *getInstruction() const { return CallSiteInst; } + + Value *getArgument(unsigned aidx) const { + assert(aidx < ArgSize && "arg index out of bounds!"); + return *(ArgBegin + aidx); + } + + bool isByValArgument(unsigned ArgNo) const { + return paramHasAttr(ArgNo + 1, Attribute::ByVal); + } + + bool doesNotThrow() const { return DoesNotThrow; } + unsigned arg_size() const { return ArgSize; } + CallSite::arg_iterator arg_begin() const { return ArgBegin; } + CallSite::arg_iterator arg_end() const { return ArgEnd; } +}; +} + +static bool InlineFunctionImpl(InlineSite IS, InlineFunctionInfo &IFI, + bool InsertLifetime, Value *&ReturnVal, + bool &InlinedMustTailCalls); + +bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI, + bool InsertLifetime) { + Value *ReturnValOut = nullptr; + bool InlinedMustTailCalls = false; + if (!InlineFunctionImpl(InlineSite(CS), IFI, InsertLifetime, ReturnValOut, + InlinedMustTailCalls)) + return false; + + Instruction *TheCall = CS.getInstruction(); + if (!TheCall->use_empty()) { + if (ReturnValOut == TheCall) // Can happen in unreachable code + ReturnValOut = UndefValue::get(TheCall->getType()); + TheCall->replaceAllUsesWith(ReturnValOut); + } + + BasicBlock *ContainingBlock = TheCall->getParent(); + + // Entire block may have become unreachable. + if (InlinedMustTailCalls && pred_empty(ContainingBlock)) + ContainingBlock->eraseFromParent(); + else + TheCall->eraseFromParent(); + + return true; +} + +namespace { /// A class for recording information about inlining through an invoke. class InvokeInliningInfo { BasicBlock *OuterResumeDest; ///< Destination of the invoke's unwind. @@ -158,6 +242,21 @@ return InnerResumeDest; } +void InlineSite::verify() const { + FunctionType *FTy = CalledFunction->getFunctionType(); + unsigned aidx = 0; + for (auto AI = ArgBegin; AI != ArgEnd; ++AI, aidx++) + assert((aidx >= FTy->getNumParams() || + FTy->getParamType(aidx) == (*AI)->getType()) && + "Calling a function with a bad signature!"); + + assert(aidx == ArgSize && "invalid ArgSize!"); +} + +Function *InlineSite::getCaller() const { + return CallSiteInst->getParent()->getParent(); +} + /// Forward the 'resume' instruction to the caller's landing pad block. /// When the landing pad block has only one predecessor, this is a simple /// branch. When there is more than one predecessor, we need to split the @@ -285,8 +384,8 @@ /// not be differentiated (and this would lead to miscompiles because the /// non-aliasing property communicated by the metadata could have /// call-site-specific control dependencies). -static void CloneAliasScopeMetadata(CallSite CS, ValueToValueMapTy &VMap) { - const Function *CalledFunc = CS.getCalledFunction(); +static void CloneAliasScopeMetadata(InlineSite IS, ValueToValueMapTy &VMap) { + const Function *CalledFunc = IS.getCalledFunction(); SetVector MD; // Note: We could only clone the metadata if it is already used in the @@ -365,12 +464,12 @@ // which instructions inside it might belong), propagate those scopes to // the inlined instructions. if (MDNode *CSM = - CS.getInstruction()->getMetadata(LLVMContext::MD_alias_scope)) + IS.getInstruction()->getMetadata(LLVMContext::MD_alias_scope)) NewMD = MDNode::concatenate(NewMD, CSM); NI->setMetadata(LLVMContext::MD_alias_scope, NewMD); } else if (NI->mayReadOrWriteMemory()) { if (MDNode *M = - CS.getInstruction()->getMetadata(LLVMContext::MD_alias_scope)) + IS.getInstruction()->getMetadata(LLVMContext::MD_alias_scope)) NI->setMetadata(LLVMContext::MD_alias_scope, M); } @@ -380,11 +479,11 @@ // which instructions inside it don't alias), propagate those scopes to // the inlined instructions. if (MDNode *CSM = - CS.getInstruction()->getMetadata(LLVMContext::MD_noalias)) + IS.getInstruction()->getMetadata(LLVMContext::MD_noalias)) NewMD = MDNode::concatenate(NewMD, CSM); NI->setMetadata(LLVMContext::MD_noalias, NewMD); } else if (NI->mayReadOrWriteMemory()) { - if (MDNode *M = CS.getInstruction()->getMetadata(LLVMContext::MD_noalias)) + if (MDNode *M = IS.getInstruction()->getMetadata(LLVMContext::MD_noalias)) NI->setMetadata(LLVMContext::MD_noalias, M); } } @@ -394,12 +493,12 @@ /// then add new alias scopes for each noalias argument, tag the mapped noalias /// parameters with noalias metadata specifying the new scope, and tag all /// non-derived loads, stores and memory intrinsics with the new alias scopes. -static void AddAliasScopeMetadata(CallSite CS, ValueToValueMapTy &VMap, +static void AddAliasScopeMetadata(InlineSite IS, ValueToValueMapTy &VMap, const DataLayout &DL, AliasAnalysis *AA) { if (!EnableNoAliasConversion) return; - const Function *CalledFunc = CS.getCalledFunction(); + const Function *CalledFunc = IS.getCalledFunction(); SmallVector NoAliasArgs; for (Function::const_arg_iterator I = CalledFunc->arg_begin(), @@ -620,37 +719,37 @@ /// If the inlined function has non-byval align arguments, then /// add @llvm.assume-based alignment assumptions to preserve this information. -static void AddAlignmentAssumptions(CallSite CS, InlineFunctionInfo &IFI) { +static void AddAlignmentAssumptions(InlineSite IS, InlineFunctionInfo &IFI) { if (!PreserveAlignmentAssumptions) return; - auto &DL = CS.getCaller()->getParent()->getDataLayout(); + auto &DL = IS.getCaller()->getParent()->getDataLayout(); // To avoid inserting redundant assumptions, we should check for assumptions // already in the caller. To do this, we might need a DT of the caller. DominatorTree DT; bool DTCalculated = false; - Function *CalledFunc = CS.getCalledFunction(); + Function *CalledFunc = IS.getCalledFunction(); for (Function::arg_iterator I = CalledFunc->arg_begin(), E = CalledFunc->arg_end(); I != E; ++I) { unsigned Align = I->getType()->isPointerTy() ? I->getParamAlignment() : 0; if (Align && !I->hasByValOrInAllocaAttr() && !I->hasNUses(0)) { if (!DTCalculated) { - DT.recalculate(const_cast(*CS.getInstruction()->getParent() - ->getParent())); + DT.recalculate(const_cast( + *IS.getInstruction()->getParent()->getParent())); DTCalculated = true; } // If we can already prove the asserted alignment in the context of the // caller, then don't bother inserting the assumption. - Value *Arg = CS.getArgument(I->getArgNo()); - if (getKnownAlignment(Arg, DL, CS.getInstruction(), + Value *Arg = IS.getArgument(I->getArgNo()); + if (getKnownAlignment(Arg, DL, IS.getInstruction(), &IFI.ACT->getAssumptionCache(*CalledFunc), &DT) >= Align) continue; - IRBuilder<>(CS.getInstruction()) + IRBuilder<>(IS.getInstruction()) .CreateAlignmentAssumption(DL, Arg, Align); } } @@ -660,13 +759,13 @@ /// update the specified callgraph to reflect the changes we made. /// Note that it's possible that not all code was copied over, so only /// some edges of the callgraph may remain. -static void UpdateCallGraphAfterInlining(CallSite CS, +static void UpdateCallGraphAfterInlining(InlineSite IS, Function::iterator FirstNewBlock, ValueToValueMapTy &VMap, InlineFunctionInfo &IFI) { CallGraph &CG = *IFI.CG; - const Function *Caller = CS.getInstruction()->getParent()->getParent(); - const Function *Callee = CS.getCalledFunction(); + const Function *Caller = IS.getInstruction()->getParent()->getParent(); + const Function *Callee = IS.getCalledFunction(); CallGraphNode *CalleeNode = CG[Callee]; CallGraphNode *CallerNode = CG[Caller]; @@ -724,7 +823,7 @@ // Update the call graph by deleting the edge from Callee to Caller. We must // do this after the loop above in case Caller and Callee are the same. - CallerNode->removeCallEdgeFor(CS); + CallerNode->removeCallEdgeFor(CallSite(IS.getInstruction())); } static void HandleByValArgumentInit(Value *Dst, Value *Src, Module *M, @@ -916,23 +1015,24 @@ /// instruction 'call B' is inlined, and 'B' calls 'C', then the call to 'C' now /// exists in the instruction stream. Similarly this will inline a recursive /// function by one level. -bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI, - bool InsertLifetime) { - Instruction *TheCall = CS.getInstruction(); +bool InlineFunctionImpl(InlineSite IS, InlineFunctionInfo &IFI, + bool InsertLifetime, Value *&ReturnVal, + bool &InlinedMustTailCalls) { + Instruction *TheCall = IS.getInstruction(); assert(TheCall->getParent() && TheCall->getParent()->getParent() && "Instruction not in function!"); // If IFI has any state in it, zap it before we fill it in. IFI.reset(); - - const Function *CalledFunc = CS.getCalledFunction(); + + const Function *CalledFunc = IS.getCalledFunction(); if (!CalledFunc || // Can't inline external function or indirect CalledFunc->isDeclaration() || // call, or call to a vararg function! CalledFunc->getFunctionType()->isVarArg()) return false; // If the call to the callee cannot throw, set the 'nounwind' flag on any // calls that we inline. - bool MarkNoUnwind = CS.doesNotThrow(); + bool MarkNoUnwind = IS.doesNotThrow(); BasicBlock *OrigBB = TheCall->getParent(); Function *Caller = OrigBB->getParent(); @@ -985,12 +1085,12 @@ auto &DL = Caller->getParent()->getDataLayout(); - assert(CalledFunc->arg_size() == CS.arg_size() && + assert(CalledFunc->arg_size() == IS.arg_size() && "No varargs calls can be inlined!"); // Calculate the vector of arguments to pass into the function cloner, which // matches up the formal to the actual argument values. - CallSite::arg_iterator AI = CS.arg_begin(); + CallSite::arg_iterator AI = IS.arg_begin(); unsigned ArgNo = 0; for (Function::const_arg_iterator I = CalledFunc->arg_begin(), E = CalledFunc->arg_end(); I != E; ++I, ++AI, ++ArgNo) { @@ -1000,7 +1100,7 @@ // by them explicit. However, we don't do this if the callee is readonly // or readnone, because the copy would be unneeded: the callee doesn't // modify the struct. - if (CS.isByValArgument(ArgNo)) { + if (IS.isByValArgument(ArgNo)) { ActualArg = HandleByValArgument(ActualArg, TheCall, CalledFunc, IFI, CalledFunc->getParamAlignment(ArgNo+1)); if (ActualArg != *AI) @@ -1013,7 +1113,7 @@ // Add alignment assumptions if necessary. We do this before the inlined // instructions are actually cloned into the caller so that we can easily // check what will be known at the start of the inlined code. - AddAlignmentAssumptions(CS, IFI); + AddAlignmentAssumptions(IS, IFI); // We want the inliner to prune the code as it copies. We would LOVE to // have no dead or constant instructions leftover after inlining occurs @@ -1033,16 +1133,16 @@ // Update the callgraph if requested. if (IFI.CG) - UpdateCallGraphAfterInlining(CS, FirstNewBlock, VMap, IFI); + UpdateCallGraphAfterInlining(IS, FirstNewBlock, VMap, IFI); // Update inlined instructions' line number information. fixupLineNumbers(Caller, FirstNewBlock, TheCall); // Clone existing noalias metadata if necessary. - CloneAliasScopeMetadata(CS, VMap); + CloneAliasScopeMetadata(IS, VMap); // Add noalias metadata if necessary. - AddAliasScopeMetadata(CS, VMap, DL, IFI.AA); + AddAliasScopeMetadata(IS, VMap, DL, IFI.AA); // FIXME: We could register any cloned assumptions instead of clearing the // whole function's cache. @@ -1095,7 +1195,7 @@ replaceDbgDeclareForAlloca(AI, AI, DIB, /*Deref=*/false); } - bool InlinedMustTailCalls = false; + InlinedMustTailCalls = false; if (InlinedFunctionInfo.ContainsCalls) { CallInst::TailCallKind CallSiteTailKind = CallInst::TCK_None; if (CallInst *CI = dyn_cast(TheCall)) @@ -1262,19 +1362,9 @@ NewBr->setDebugLoc(Returns[0]->getDebugLoc()); } - // If the return instruction returned a value, replace uses of the call with - // uses of the returned value. - if (!TheCall->use_empty()) { - ReturnInst *R = Returns[0]; - if (TheCall == R->getReturnValue()) - TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType())); - else - TheCall->replaceAllUsesWith(R->getReturnValue()); - } - // Since we are now done with the Call/Invoke, we can delete it. - TheCall->eraseFromParent(); + ReturnVal = Returns[0]->getReturnValue(); - // Since we are now done with the return instruction, delete it also. + // Since we are now done with the return instruction, delete it. Returns[0]->eraseFromParent(); // We are now done with the inlining. @@ -1336,7 +1426,7 @@ AfterCallBB->begin()); // Anything that used the result of the function call should now use the // PHI node as their operand. - TheCall->replaceAllUsesWith(PHI); + ReturnVal = PHI; } // Loop over all of the return instructions adding entries to the PHI node @@ -1367,14 +1457,9 @@ if (CreatedBranchToNormalDest) CreatedBranchToNormalDest->setDebugLoc(Loc); } else if (!Returns.empty()) { - // Otherwise, if there is exactly one return value, just replace anything - // using the return value of the call with the computed value. - if (!TheCall->use_empty()) { - if (TheCall == Returns[0]->getReturnValue()) - TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType())); - else - TheCall->replaceAllUsesWith(Returns[0]->getReturnValue()); - } + // If there is exactly one return value, we do not need to introduce PHI + // nodes. + ReturnVal = Returns[0]->getReturnValue(); // Update PHI nodes that use the ReturnBB to use the AfterCallBB. BasicBlock *ReturnBB = Returns[0]->getParent(); @@ -1391,20 +1476,11 @@ // Delete the return instruction now and empty ReturnBB now. Returns[0]->eraseFromParent(); ReturnBB->eraseFromParent(); - } else if (!TheCall->use_empty()) { - // No returns, but something is using the return value of the call. Just - // nuke the result. - TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType())); + } else { + // No returns, hence no well-defined return value + ReturnVal = UndefValue::get(TheCall->getType()); } - // Since we are now done with the Call/Invoke, we can delete it. - TheCall->eraseFromParent(); - - // If we inlined any musttail calls and the original return is now - // unreachable, delete it. It can only contain a bitcast and ret. - if (InlinedMustTailCalls && pred_begin(AfterCallBB) == pred_end(AfterCallBB)) - AfterCallBB->eraseFromParent(); - // We should always be able to fold the entry block of the function into the // single predecessor of the block... assert(cast(Br)->isUnconditional() && "splitBasicBlock broken!"); @@ -1430,6 +1506,9 @@ &IFI.ACT->getAssumptionCache(*Caller))) { PHI->replaceAllUsesWith(V); PHI->eraseFromParent(); + assert(ReturnVal == PHI && + "PHI was created to represent the return value!"); + ReturnVal = V; } }