Index: lib/Analysis/InlineCost.cpp =================================================================== --- lib/Analysis/InlineCost.cpp +++ lib/Analysis/InlineCost.cpp @@ -109,6 +109,9 @@ bool HasReturn; bool HasIndirectBr; bool HasFrameEscape; + bool HasVaStart; // whether F has va_start in it. + bool HasMustTailCall; // whether F has a tail or musttail call in it. + bool FIsVarArg; // whether F is variadic by declaration. /// Number of bytes allocated statically by the callee. uint64_t AllocatedSize; @@ -225,7 +228,8 @@ Cost(0), IsCallerRecursive(false), IsRecursiveCall(false), ExposesReturnsTwice(false), HasDynamicAlloca(false), ContainsNoDuplicateCall(false), HasReturn(false), HasIndirectBr(false), - HasFrameEscape(false), AllocatedSize(0), NumInstructions(0), + HasFrameEscape(false), HasVaStart(false), HasMustTailCall(false), + FIsVarArg(Callee.isVarArg()), AllocatedSize(0), NumInstructions(0), NumVectorInstructions(0), FiftyPercentVectorBonus(0), TenPercentVectorBonus(0), VectorBonus(0), NumConstantArgs(0), NumConstantOffsetPtrArgs(0), NumAllocaArgs(0), NumConstantPtrCmps(0), @@ -902,6 +906,17 @@ } if (CS.isCall() && cast<CallInst>(CS.getInstruction())->cannotDuplicate()) ContainsNoDuplicateCall = true; + if (CS.isCall()) { + auto CI = cast<CallInst>(CS.getInstruction()); + if (CI->isMustTailCall()) { + HasMustTailCall = true; + // If A calls a variadic function B and B has a musttail call to C, then + // C can do va_start and access args passed from A to B against '...'. + // Therefore we can not inline B inside A. + if (FIsVarArg) + return false; // No point looking further. + } + } if (Function *F = CS.getCalledFunction()) { // When we have a concrete function, first try to simplify it directly. @@ -928,6 +943,10 @@ case Intrinsic::localescape: HasFrameEscape = true; return false; + case Intrinsic::vastart: + // The callee has a va_start. We can not handle that in inlinining. + HasVaStart = true; + return false; } } @@ -1184,7 +1203,8 @@ // If the visit this instruction detected an uninlinable pattern, abort. if (IsRecursiveCall || ExposesReturnsTwice || HasDynamicAlloca || - HasIndirectBr || HasFrameEscape) + HasIndirectBr || HasFrameEscape || HasVaStart || + (HasMustTailCall && FIsVarArg)) return false; // If the caller is a recursive function then we don't want to inline @@ -1566,6 +1586,7 @@ bool llvm::isInlineViable(Function &F) { bool ReturnsTwice = F.hasFnAttribute(Attribute::ReturnsTwice); + bool IsVarArg = F.isVarArg(); for (Function::iterator BI = F.begin(), BE = F.end(); BI != BE; ++BI) { // Disallow inlining of functions which contain indirect branches or // blockaddresses. @@ -1577,6 +1598,14 @@ if (!CS) continue; + if (IsVarArg) { + if (const CallInst *CI = dyn_cast<CallInst>(&II)) { + // Same logic as in analyzeBlock above. + if (CI->isMustTailCall()) + return false; + } + } + // Disallow recursive calls. if (&F == CS.getCalledFunction()) return false; @@ -1587,12 +1616,13 @@ cast<CallInst>(CS.getInstruction())->canReturnTwice()) return false; - // Disallow inlining functions that call @llvm.localescape. Doing this - // correctly would require major changes to the inliner. - if (CS.getCalledFunction() && - CS.getCalledFunction()->getIntrinsicID() == - llvm::Intrinsic::localescape) - return false; + // Disallow inlining functions that call @llvm.localescape or @vastart. + // Doing this correctly would require major changes to the inliner. + if (auto CF = CS.getCalledFunction()) { + auto IntrId = CF->getIntrinsicID(); + if (IntrId == Intrinsic::localescape || IntrId == Intrinsic::vastart) + return false; + } } } Index: lib/Transforms/Utils/InlineFunction.cpp =================================================================== --- lib/Transforms/Utils/InlineFunction.cpp +++ lib/Transforms/Utils/InlineFunction.cpp @@ -1463,9 +1463,9 @@ IFI.reset(); Function *CalledFunc = CS.getCalledFunction(); - if (!CalledFunc || // Can't inline external function or indirect - CalledFunc->isDeclaration() || // call, or call to a vararg function! - CalledFunc->getFunctionType()->isVarArg()) return false; + if (!CalledFunc || // Can't inline external function or indirect calls + CalledFunc->isDeclaration()) + return false; // The inliner does not know how to inline through calls with operand bundles // in general ... @@ -1592,8 +1592,12 @@ auto &DL = Caller->getParent()->getDataLayout(); - assert(CalledFunc->arg_size() == CS.arg_size() && - "No varargs calls can be inlined!"); + if (CalledFunc->getFunctionType()->isVarArg()) + assert(CalledFunc->arg_size() <= CS.arg_size() && + "Not enough arguments passed to vararg function"); + else + assert(CalledFunc->arg_size() == CS.arg_size() && + "No varargs calls can be inlined!"); // Calculate the vector of arguments to pass into the function cloner, which // matches up the formal to the actual argument values. @@ -1781,16 +1785,25 @@ if (CallInst *CI = dyn_cast<CallInst>(TheCall)) CallSiteTailKind = CI->getTailCallKind(); + bool CalleeIsVararg = CalledFunc->isVarArg(); // Just for asserts below. for (Function::iterator BB = FirstNewBlock, E = Caller->end(); BB != E; ++BB) { for (Instruction &I : *BB) { CallInst *CI = dyn_cast<CallInst>(&I); if (!CI) continue; + if (CalleeIsVararg) { + assert((!CI->isMustTailCall()) && + "Inlined musttailcall in vararg function"); + } - if (Function *F = CI->getCalledFunction()) + if (Function *F = CI->getCalledFunction()) { InlinedDeoptimizeCalls |= F->getIntrinsicID() == Intrinsic::experimental_deoptimize; + if (CalleeIsVararg) + assert((F->getIntrinsicID() != Intrinsic::vastart) && + "Inlined vastart"); + } // We need to reduce the strength of any inlined tail calls. For // musttail, we have to avoid introducing potential unbounded stack