Index: lib/Transforms/Utils/InlineFunction.cpp =================================================================== --- lib/Transforms/Utils/InlineFunction.cpp +++ lib/Transforms/Utils/InlineFunction.cpp @@ -63,6 +63,68 @@ } namespace { +class InlineSite { + CallSite CS; + + bool paramHasAttr(unsigned i, Attribute::AttrKind A) const { + return CS.paramHasAttr(i, A); + } + +public: + explicit InlineSite(CallSite CS) : CS(CS) { } + + Function *getCalledFunction() const { return CS.getCalledFunction(); } + + Function *getCaller() const { return CS.getCaller(); } + + Instruction *getInstruction() const { return CS.getInstruction(); } + + Value *getArgument(unsigned Index) const { + return CS.getArgument(Index); + } + + bool isByValArgument(unsigned ArgNo) const { + return paramHasAttr(ArgNo + 1, Attribute::ByVal); + } + + bool doesNotThrow() const { return CS.doesNotThrow(); } + unsigned arg_size() const { return CS.arg_size(); } + CallSite::arg_iterator arg_begin() const { return CS.arg_begin(); } + CallSite::arg_iterator arg_end() const { return CS.arg_end(); } +}; +} + +static bool InlineThroughInlineSite(InlineSite IS, InlineFunctionInfo &IFI, + bool InsertLifetime, Value *&ReturnVal, + bool &InlinedMustTailCalls); + +bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI, + bool InsertLifetime) { + Value *ReturnValOut = nullptr; + bool InlinedMustTailCalls = false; + if (!InlineThroughInlineSite(InlineSite(CS), IFI, InsertLifetime, + ReturnValOut, InlinedMustTailCalls)) + return false; + + Instruction *TheCall = CS.getInstruction(); + if (!TheCall->use_empty()) { + if (ReturnValOut == TheCall) // Can happen in unreachable code + ReturnValOut = UndefValue::get(TheCall->getType()); + TheCall->replaceAllUsesWith(ReturnValOut); + } + + BasicBlock *ContainingBlock = TheCall->getParent(); + + // Entire block may have become unreachable. + if (InlinedMustTailCalls && pred_empty(ContainingBlock)) + ContainingBlock->eraseFromParent(); + else + TheCall->eraseFromParent(); + + return true; +} + +namespace { /// A class for recording information about inlining through an invoke. class InvokeInliningInfo { BasicBlock *OuterResumeDest; ///< Destination of the invoke's unwind. @@ -285,8 +347,8 @@ /// not be differentiated (and this would lead to miscompiles because the /// non-aliasing property communicated by the metadata could have /// call-site-specific control dependencies). -static void CloneAliasScopeMetadata(CallSite CS, ValueToValueMapTy &VMap) { - const Function *CalledFunc = CS.getCalledFunction(); +static void CloneAliasScopeMetadata(InlineSite IS, ValueToValueMapTy &VMap) { + const Function *CalledFunc = IS.getCalledFunction(); SetVector MD; // Note: We could only clone the metadata if it is already used in the @@ -365,12 +427,12 @@ // which instructions inside it might belong), propagate those scopes to // the inlined instructions. if (MDNode *CSM = - CS.getInstruction()->getMetadata(LLVMContext::MD_alias_scope)) + IS.getInstruction()->getMetadata(LLVMContext::MD_alias_scope)) NewMD = MDNode::concatenate(NewMD, CSM); NI->setMetadata(LLVMContext::MD_alias_scope, NewMD); } else if (NI->mayReadOrWriteMemory()) { if (MDNode *M = - CS.getInstruction()->getMetadata(LLVMContext::MD_alias_scope)) + IS.getInstruction()->getMetadata(LLVMContext::MD_alias_scope)) NI->setMetadata(LLVMContext::MD_alias_scope, M); } @@ -380,11 +442,11 @@ // which instructions inside it don't alias), propagate those scopes to // the inlined instructions. if (MDNode *CSM = - CS.getInstruction()->getMetadata(LLVMContext::MD_noalias)) + IS.getInstruction()->getMetadata(LLVMContext::MD_noalias)) NewMD = MDNode::concatenate(NewMD, CSM); NI->setMetadata(LLVMContext::MD_noalias, NewMD); } else if (NI->mayReadOrWriteMemory()) { - if (MDNode *M = CS.getInstruction()->getMetadata(LLVMContext::MD_noalias)) + if (MDNode *M = IS.getInstruction()->getMetadata(LLVMContext::MD_noalias)) NI->setMetadata(LLVMContext::MD_noalias, M); } } @@ -394,12 +456,12 @@ /// then add new alias scopes for each noalias argument, tag the mapped noalias /// parameters with noalias metadata specifying the new scope, and tag all /// non-derived loads, stores and memory intrinsics with the new alias scopes. -static void AddAliasScopeMetadata(CallSite CS, ValueToValueMapTy &VMap, +static void AddAliasScopeMetadata(InlineSite IS, ValueToValueMapTy &VMap, const DataLayout &DL, AliasAnalysis *AA) { if (!EnableNoAliasConversion) return; - const Function *CalledFunc = CS.getCalledFunction(); + const Function *CalledFunc = IS.getCalledFunction(); SmallVector NoAliasArgs; for (Function::const_arg_iterator I = CalledFunc->arg_begin(), @@ -620,37 +682,37 @@ /// If the inlined function has non-byval align arguments, then /// add @llvm.assume-based alignment assumptions to preserve this information. -static void AddAlignmentAssumptions(CallSite CS, InlineFunctionInfo &IFI) { +static void AddAlignmentAssumptions(InlineSite IS, InlineFunctionInfo &IFI) { if (!PreserveAlignmentAssumptions) return; - auto &DL = CS.getCaller()->getParent()->getDataLayout(); + auto &DL = IS.getCaller()->getParent()->getDataLayout(); // To avoid inserting redundant assumptions, we should check for assumptions // already in the caller. To do this, we might need a DT of the caller. DominatorTree DT; bool DTCalculated = false; - Function *CalledFunc = CS.getCalledFunction(); + Function *CalledFunc = IS.getCalledFunction(); for (Function::arg_iterator I = CalledFunc->arg_begin(), E = CalledFunc->arg_end(); I != E; ++I) { unsigned Align = I->getType()->isPointerTy() ? I->getParamAlignment() : 0; if (Align && !I->hasByValOrInAllocaAttr() && !I->hasNUses(0)) { if (!DTCalculated) { - DT.recalculate(const_cast(*CS.getInstruction()->getParent() - ->getParent())); + DT.recalculate(const_cast( + *IS.getInstruction()->getParent()->getParent())); DTCalculated = true; } // If we can already prove the asserted alignment in the context of the // caller, then don't bother inserting the assumption. - Value *Arg = CS.getArgument(I->getArgNo()); - if (getKnownAlignment(Arg, DL, CS.getInstruction(), + Value *Arg = IS.getArgument(I->getArgNo()); + if (getKnownAlignment(Arg, DL, IS.getInstruction(), &IFI.ACT->getAssumptionCache(*CalledFunc), &DT) >= Align) continue; - IRBuilder<>(CS.getInstruction()) + IRBuilder<>(IS.getInstruction()) .CreateAlignmentAssumption(DL, Arg, Align); } } @@ -660,13 +722,13 @@ /// update the specified callgraph to reflect the changes we made. /// Note that it's possible that not all code was copied over, so only /// some edges of the callgraph may remain. -static void UpdateCallGraphAfterInlining(CallSite CS, +static void UpdateCallGraphAfterInlining(InlineSite IS, Function::iterator FirstNewBlock, ValueToValueMapTy &VMap, InlineFunctionInfo &IFI) { CallGraph &CG = *IFI.CG; - const Function *Caller = CS.getInstruction()->getParent()->getParent(); - const Function *Callee = CS.getCalledFunction(); + const Function *Caller = IS.getInstruction()->getParent()->getParent(); + const Function *Callee = IS.getCalledFunction(); CallGraphNode *CalleeNode = CG[Callee]; CallGraphNode *CallerNode = CG[Caller]; @@ -724,7 +786,7 @@ // Update the call graph by deleting the edge from Callee to Caller. We must // do this after the loop above in case Caller and Callee are the same. - CallerNode->removeCallEdgeFor(CS); + CallerNode->removeCallEdgeFor(CallSite(IS.getInstruction())); } static void HandleByValArgumentInit(Value *Dst, Value *Src, Module *M, @@ -916,23 +978,24 @@ /// instruction 'call B' is inlined, and 'B' calls 'C', then the call to 'C' now /// exists in the instruction stream. Similarly this will inline a recursive /// function by one level. -bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI, - bool InsertLifetime) { - Instruction *TheCall = CS.getInstruction(); +bool InlineThroughInlineSite(InlineSite IS, InlineFunctionInfo &IFI, + bool InsertLifetime, Value *&ReturnVal, + bool &InlinedMustTailCalls) { + Instruction *TheCall = IS.getInstruction(); assert(TheCall->getParent() && TheCall->getParent()->getParent() && "Instruction not in function!"); // If IFI has any state in it, zap it before we fill it in. IFI.reset(); - - const Function *CalledFunc = CS.getCalledFunction(); + + const Function *CalledFunc = IS.getCalledFunction(); if (!CalledFunc || // Can't inline external function or indirect CalledFunc->isDeclaration() || // call, or call to a vararg function! CalledFunc->getFunctionType()->isVarArg()) return false; // If the call to the callee cannot throw, set the 'nounwind' flag on any // calls that we inline. - bool MarkNoUnwind = CS.doesNotThrow(); + bool MarkNoUnwind = IS.doesNotThrow(); BasicBlock *OrigBB = TheCall->getParent(); Function *Caller = OrigBB->getParent(); @@ -985,12 +1048,12 @@ auto &DL = Caller->getParent()->getDataLayout(); - assert(CalledFunc->arg_size() == CS.arg_size() && + assert(CalledFunc->arg_size() == IS.arg_size() && "No varargs calls can be inlined!"); // Calculate the vector of arguments to pass into the function cloner, which // matches up the formal to the actual argument values. - CallSite::arg_iterator AI = CS.arg_begin(); + CallSite::arg_iterator AI = IS.arg_begin(); unsigned ArgNo = 0; for (Function::const_arg_iterator I = CalledFunc->arg_begin(), E = CalledFunc->arg_end(); I != E; ++I, ++AI, ++ArgNo) { @@ -1000,7 +1063,7 @@ // by them explicit. However, we don't do this if the callee is readonly // or readnone, because the copy would be unneeded: the callee doesn't // modify the struct. - if (CS.isByValArgument(ArgNo)) { + if (IS.isByValArgument(ArgNo)) { ActualArg = HandleByValArgument(ActualArg, TheCall, CalledFunc, IFI, CalledFunc->getParamAlignment(ArgNo+1)); if (ActualArg != *AI) @@ -1013,7 +1076,7 @@ // Add alignment assumptions if necessary. We do this before the inlined // instructions are actually cloned into the caller so that we can easily // check what will be known at the start of the inlined code. - AddAlignmentAssumptions(CS, IFI); + AddAlignmentAssumptions(IS, IFI); // We want the inliner to prune the code as it copies. We would LOVE to // have no dead or constant instructions leftover after inlining occurs @@ -1033,16 +1096,16 @@ // Update the callgraph if requested. if (IFI.CG) - UpdateCallGraphAfterInlining(CS, FirstNewBlock, VMap, IFI); + UpdateCallGraphAfterInlining(IS, FirstNewBlock, VMap, IFI); // Update inlined instructions' line number information. fixupLineNumbers(Caller, FirstNewBlock, TheCall); // Clone existing noalias metadata if necessary. - CloneAliasScopeMetadata(CS, VMap); + CloneAliasScopeMetadata(IS, VMap); // Add noalias metadata if necessary. - AddAliasScopeMetadata(CS, VMap, DL, IFI.AA); + AddAliasScopeMetadata(IS, VMap, DL, IFI.AA); // FIXME: We could register any cloned assumptions instead of clearing the // whole function's cache. @@ -1095,7 +1158,7 @@ replaceDbgDeclareForAlloca(AI, AI, DIB, /*Deref=*/false); } - bool InlinedMustTailCalls = false; + InlinedMustTailCalls = false; if (InlinedFunctionInfo.ContainsCalls) { CallInst::TailCallKind CallSiteTailKind = CallInst::TCK_None; if (CallInst *CI = dyn_cast(TheCall)) @@ -1262,19 +1325,9 @@ NewBr->setDebugLoc(Returns[0]->getDebugLoc()); } - // If the return instruction returned a value, replace uses of the call with - // uses of the returned value. - if (!TheCall->use_empty()) { - ReturnInst *R = Returns[0]; - if (TheCall == R->getReturnValue()) - TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType())); - else - TheCall->replaceAllUsesWith(R->getReturnValue()); - } - // Since we are now done with the Call/Invoke, we can delete it. - TheCall->eraseFromParent(); + ReturnVal = Returns[0]->getReturnValue(); - // Since we are now done with the return instruction, delete it also. + // Since we are now done with the return instruction, delete it. Returns[0]->eraseFromParent(); // We are now done with the inlining. @@ -1336,7 +1389,7 @@ AfterCallBB->begin()); // Anything that used the result of the function call should now use the // PHI node as their operand. - TheCall->replaceAllUsesWith(PHI); + ReturnVal = PHI; } // Loop over all of the return instructions adding entries to the PHI node @@ -1367,14 +1420,9 @@ if (CreatedBranchToNormalDest) CreatedBranchToNormalDest->setDebugLoc(Loc); } else if (!Returns.empty()) { - // Otherwise, if there is exactly one return value, just replace anything - // using the return value of the call with the computed value. - if (!TheCall->use_empty()) { - if (TheCall == Returns[0]->getReturnValue()) - TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType())); - else - TheCall->replaceAllUsesWith(Returns[0]->getReturnValue()); - } + // If there is exactly one return value, we do not need to introduce PHI + // nodes. + ReturnVal = Returns[0]->getReturnValue(); // Update PHI nodes that use the ReturnBB to use the AfterCallBB. BasicBlock *ReturnBB = Returns[0]->getParent(); @@ -1391,20 +1439,11 @@ // Delete the return instruction now and empty ReturnBB now. Returns[0]->eraseFromParent(); ReturnBB->eraseFromParent(); - } else if (!TheCall->use_empty()) { - // No returns, but something is using the return value of the call. Just - // nuke the result. - TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType())); + } else { + // No returns, hence no well-defined return value + ReturnVal = UndefValue::get(TheCall->getType()); } - // Since we are now done with the Call/Invoke, we can delete it. - TheCall->eraseFromParent(); - - // If we inlined any musttail calls and the original return is now - // unreachable, delete it. It can only contain a bitcast and ret. - if (InlinedMustTailCalls && pred_begin(AfterCallBB) == pred_end(AfterCallBB)) - AfterCallBB->eraseFromParent(); - // We should always be able to fold the entry block of the function into the // single predecessor of the block... assert(cast(Br)->isUnconditional() && "splitBasicBlock broken!"); @@ -1430,6 +1469,9 @@ &IFI.ACT->getAssumptionCache(*Caller))) { PHI->replaceAllUsesWith(V); PHI->eraseFromParent(); + assert(ReturnVal == PHI && + "PHI was created to represent the return value!"); + ReturnVal = V; } }