Index: llvm/docs/LangRef.rst =================================================================== --- llvm/docs/LangRef.rst +++ llvm/docs/LangRef.rst @@ -1401,6 +1401,11 @@ inlining this function is desirable (such as the "inline" keyword in C/C++). It is just a hint; it imposes no requirements on the inliner. +``inline_va_arg_pack`` + This attribute can appear on a call or invoke in a function with + available_externally linkage. If the function is inlined, the variadic + arguments of the caller will be forwarded into the call instruction. This + attribute is used to implement `__builtin_va_arg_pack`. ``jumptable`` This attribute indicates that the function should be added to a jump-instruction table at code-generation time, and that all address-taken @@ -10445,6 +10450,30 @@ intrinsic is necessary because the `` llvm.va_start`` intrinsic may be arbitrarily complex and require, for example, memory allocation. +'``llvm.va_arg_pack_len``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare i32 @llvm.va_arg_pack_len() + +Overview: +""""""""" + +The '``llvm.va_arg_pack_len``' intrinsic folds to the number of variadic +arguments to the current function if its enclosing function is inlined. + +Semantics: +"""""""""" + +This intrinsic can only appear in an ``available_externally`` variadic function. +As the enclosing function is being inlined, the inliner folds the call to the +difference of the total number of arguments at the call site and the number of +declared parameters. + Accurate Garbage Collection Intrinsics -------------------------------------- Index: llvm/include/llvm/Bitcode/LLVMBitCodes.h =================================================================== --- llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -603,6 +603,7 @@ ATTR_KIND_OPT_FOR_FUZZING = 57, ATTR_KIND_SHADOWCALLSTACK = 58, ATTR_KIND_SPECULATIVE_LOAD_HARDENING = 59, + ATTR_KIND_INLINE_VA_ARG_PACK = 60, }; enum ComdatSelectionKindCodes { Index: llvm/include/llvm/IR/Attributes.td =================================================================== --- llvm/include/llvm/IR/Attributes.td +++ llvm/include/llvm/IR/Attributes.td @@ -207,6 +207,12 @@ def NoJumpTables : StrBoolAttr<"no-jump-tables">; def ProfileSampleAccurate : StrBoolAttr<"profile-sample-accurate">; +/// This applies to a CallInst to a function whose last argument in C was a call +/// to builtin_va_arg_pack(). When inlining, the vaargs call arguments are +/// appended to the call arguments of this function, and the attribute is +/// stripped. +def InlineVAArgPack : EnumAttr<"inline_va_arg_pack">; + class CompatRule { // The name of the function called to check the attribute of the caller and // callee and decide whether inlining should be allowed. The function's Index: llvm/include/llvm/IR/InstrTypes.h =================================================================== --- llvm/include/llvm/IR/InstrTypes.h +++ llvm/include/llvm/IR/InstrTypes.h @@ -1601,6 +1601,19 @@ removeAttribute(AttributeList::FunctionIndex, Attribute::Convergent); } + /// Determine if we should forward variadic arguments into this call when + /// inlining. + bool isInlineVAArgPackCall() const { + return Attrs.hasAttribute(AttributeList::FunctionIndex, + Attribute::InlineVAArgPack); + } + void setIsInlineVAArgPackCall(bool Val) { + if (Val) + addAttribute(AttributeList::FunctionIndex, Attribute::InlineVAArgPack); + else + removeAttribute(AttributeList::FunctionIndex, Attribute::InlineVAArgPack); + } + /// Determine if the call returns a structure through first /// pointer argument. bool hasStructRetAttr() const { Index: llvm/include/llvm/IR/Intrinsics.td =================================================================== --- llvm/include/llvm/IR/Intrinsics.td +++ llvm/include/llvm/IR/Intrinsics.td @@ -312,6 +312,9 @@ "llvm.va_copy">; def int_vaend : Intrinsic<[], [llvm_ptr_ty], [], "llvm.va_end">; +def int_va_arg_pack_len : Intrinsic<[llvm_i32_ty], [], [], + "llvm.va_arg_pack_len">; + //===------------------- Garbage Collection Intrinsics --------------------===// // def int_gcroot : Intrinsic<[], Index: llvm/lib/AsmParser/LLLexer.cpp =================================================================== --- llvm/lib/AsmParser/LLLexer.cpp +++ llvm/lib/AsmParser/LLLexer.cpp @@ -640,6 +640,7 @@ KEYWORD(inaccessiblememonly); KEYWORD(inaccessiblemem_or_argmemonly); KEYWORD(inlinehint); + KEYWORD(inline_va_arg_pack); KEYWORD(inreg); KEYWORD(jumptable); KEYWORD(minsize); Index: llvm/lib/AsmParser/LLParser.cpp =================================================================== --- llvm/lib/AsmParser/LLParser.cpp +++ llvm/lib/AsmParser/LLParser.cpp @@ -1237,6 +1237,8 @@ case lltok::kw_inaccessiblemem_or_argmemonly: B.addAttribute(Attribute::InaccessibleMemOrArgMemOnly); break; case lltok::kw_inlinehint: B.addAttribute(Attribute::InlineHint); break; + case lltok::kw_inline_va_arg_pack: + B.addAttribute(Attribute::InlineVAArgPack); break; case lltok::kw_jumptable: B.addAttribute(Attribute::JumpTable); break; case lltok::kw_minsize: B.addAttribute(Attribute::MinSize); break; case lltok::kw_naked: B.addAttribute(Attribute::Naked); break; Index: llvm/lib/AsmParser/LLToken.h =================================================================== --- llvm/lib/AsmParser/LLToken.h +++ llvm/lib/AsmParser/LLToken.h @@ -184,6 +184,7 @@ kw_inaccessiblememonly, kw_inaccessiblemem_or_argmemonly, kw_inlinehint, + kw_inline_va_arg_pack, kw_inreg, kw_jumptable, kw_minsize, Index: llvm/lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -1188,6 +1188,7 @@ case Attribute::ShadowCallStack: return 1ULL << 59; case Attribute::SpeculativeLoadHardening: return 1ULL << 60; + case Attribute::InlineVAArgPack: return 1ULL << 61; case Attribute::Dereferenceable: llvm_unreachable("dereferenceable attribute not supported in raw format"); break; @@ -1328,6 +1329,8 @@ return Attribute::InaccessibleMemOrArgMemOnly; case bitc::ATTR_KIND_INLINE_HINT: return Attribute::InlineHint; + case bitc::ATTR_KIND_INLINE_VA_ARG_PACK: + return Attribute::InlineVAArgPack; case bitc::ATTR_KIND_IN_REG: return Attribute::InReg; case bitc::ATTR_KIND_JUMP_TABLE: Index: llvm/lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -618,6 +618,8 @@ return bitc::ATTR_KIND_INACCESSIBLEMEM_OR_ARGMEMONLY; case Attribute::InlineHint: return bitc::ATTR_KIND_INLINE_HINT; + case Attribute::InlineVAArgPack: + return bitc::ATTR_KIND_INLINE_VA_ARG_PACK; case Attribute::InReg: return bitc::ATTR_KIND_IN_REG; case Attribute::JumpTable: Index: llvm/lib/IR/Attributes.cpp =================================================================== --- llvm/lib/IR/Attributes.cpp +++ llvm/lib/IR/Attributes.cpp @@ -268,6 +268,8 @@ return "inalloca"; if (hasAttribute(Attribute::InlineHint)) return "inlinehint"; + if (hasAttribute(Attribute::InlineVAArgPack)) + return "inline_va_arg_pack"; if (hasAttribute(Attribute::InReg)) return "inreg"; if (hasAttribute(Attribute::JumpTable)) Index: llvm/lib/IR/Verifier.cpp =================================================================== --- llvm/lib/IR/Verifier.cpp +++ llvm/lib/IR/Verifier.cpp @@ -1512,6 +1512,7 @@ case Attribute::NoRecurse: case Attribute::InaccessibleMemOnly: case Attribute::InaccessibleMemOrArgMemOnly: + case Attribute::InlineVAArgPack: case Attribute::AllocSize: case Attribute::SpeculativeLoadHardening: case Attribute::Speculatable: @@ -1530,6 +1531,10 @@ Kind == Attribute::ReadNone; } +static bool isCallOnlyAttr(Attribute::AttrKind Kind) { + return Kind == Attribute::InlineVAArgPack; +} + void Verifier::verifyAttributeTypes(AttributeSet Attrs, bool IsFunction, const Value *V) { for (Attribute A : Attrs) { @@ -1548,6 +1553,11 @@ "' does not apply to functions!", V); return; + } else if (isCallOnlyAttr(A.getKindAsEnum()) && + (!isa(V) || !IsFunction)) { + CheckFailed("Attribute '" + A.getAsString() + + "' only applies to call instructions"); + return; } } } @@ -3023,6 +3033,15 @@ if (CI.isMustTailCall()) verifyMustTailCall(CI); + + if (CI.isInlineVAArgPackCall()) { + Function *FD = CI.getParent()->getParent(); + Assert( + FD->hasAvailableExternallyLinkage(), + "inline_va_arg_pack must be used in an available_externally function"); + Assert(FD->isVarArg(), + "inline_va_arg_pack can only appear in a variadic function"); + } } void Verifier::visitInvokeInst(InvokeInst &II) { @@ -4592,6 +4611,16 @@ "the scale of smul_fix must be less than the width of the operands"); break; } + case Intrinsic::va_arg_pack_len: { + Function *FD = Call.getParent()->getParent(); + Assert(FD->hasAvailableExternallyLinkage(), + "call to @llvm.va_arg_pack_len can only appear in a function with" + "available_externally linkage"); + Assert( + FD->isVarArg(), + "call to @llvm.va_arg_pack_len can only appear in a variadic function"); + break; + } }; } Index: llvm/lib/Transforms/Utils/CodeExtractor.cpp =================================================================== --- llvm/lib/Transforms/Utils/CodeExtractor.cpp +++ llvm/lib/Transforms/Utils/CodeExtractor.cpp @@ -181,7 +181,14 @@ // proposals for fixing this in llvm.org/PR39545. if (IID == Intrinsic::eh_typeid_for) return false; + + // Calls to this intrinsic are tied to the current function. + if (IID == Intrinsic::va_arg_pack_len) + return false; } + // Calls with this attribute are tied to the current function. + if (CI->hasFnAttr(Attribute::InlineVAArgPack)) + return false; } } @@ -824,6 +831,8 @@ case Attribute::UWTable: case Attribute::NoCfCheck: break; + case Attribute::InlineVAArgPack: + llvm_unreachable("InlineVAArgPack should never reach here"); } newFunction->addFnAttr(Attr); Index: llvm/lib/Transforms/Utils/InlineFunction.cpp =================================================================== --- llvm/lib/Transforms/Utils/InlineFunction.cpp +++ llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -1824,6 +1824,72 @@ VarArgsAttrs.push_back(CS.getAttributes().getParamAttributes(i)); } + auto forwardVAArgsToCall = [&](CallBase *CI) { + // Collect attributes for non-vararg parameters. + AttributeList Attrs = CI->getAttributes(); + SmallVector ArgAttrs; + if (!Attrs.isEmpty() || !VarArgsAttrs.empty()) { + for (unsigned ArgNo = 0; ArgNo < CI->getFunctionType()->getNumParams(); + ++ArgNo) + ArgAttrs.push_back(Attrs.getParamAttributes(ArgNo)); + } + + // Add VarArg attributes. + ArgAttrs.append(VarArgsAttrs.begin(), VarArgsAttrs.end()); + Attrs = AttributeList::get(CI->getContext(), Attrs.getFnAttributes(), + Attrs.getRetAttributes(), ArgAttrs); + // Add VarArgs to existing parameters. + SmallVector Params(CI->arg_operands()); + Params.append(VarArgsToForward.begin(), VarArgsToForward.end()); + + CallBase *NewCI = nullptr; + // Rebuild the call/invoke. + if (auto *Call = dyn_cast(CI)) { + NewCI = CallInst::Create(Call->getFunctionType(), + Call->getCalledOperand(), Params, "", Call); + } else if (auto *Invoke = dyn_cast(CI)) { + NewCI = InvokeInst::Create( + Invoke->getFunctionType(), Invoke->getCalledOperand(), + Invoke->getNormalDest(), Invoke->getUnwindDest(), Params, "", Invoke); + } else { + llvm_unreachable("CallBase that isn't call or invoke?"); + } + + NewCI->setDebugLoc(CI->getDebugLoc()); + NewCI->setAttributes(Attrs); + NewCI->setCallingConv(CI->getCallingConv()); + return NewCI; + }; + + // Walk through the newly inserted basic blocks, replacing calls to + // @llvm.va_arg_pack_len with the number of variadic arguments, and expanding + // variadic arguments into calls annotated with 'inline_va_arg_pack'. + for (Function::iterator B = FirstNewBlock, E = Caller->end(); B != E; ++B) { + for (auto II = B->begin(); II != B->end();) { + Instruction *I = &*II++; + if (CallBase *CB = dyn_cast(I)) { + if (CB->isInlineVAArgPackCall()) { + auto *NewCB = forwardVAArgsToCall(CB); + NewCB->setIsInlineVAArgPackCall(false); + CB->replaceAllUsesWith(NewCB); + CB->eraseFromParent(); + continue; + } + + Function *CalledFunction = CB->getCalledFunction(); + if (CalledFunction && + CalledFunction->getIntrinsicID() == Intrinsic::va_arg_pack_len) { + unsigned Val = CS.getNumArgOperands() - + CalledFunc->getFunctionType()->getNumParams(); + Value *Result = ConstantInt::get(CB->getType(), Val); + CB->replaceAllUsesWith(Result); + CB->eraseFromParent(); + continue; + } + } + } + } + bool InlinedMustTailCalls = false, InlinedDeoptimizeCalls = false; if (InlinedFunctionInfo.ContainsCalls) { CallInst::TailCallKind CallSiteTailKind = CallInst::TCK_None; @@ -1848,27 +1914,7 @@ ((ForwardVarArgsTo && CI->getCalledFunction() == ForwardVarArgsTo) || CI->isMustTailCall())) { - // Collect attributes for non-vararg parameters. - AttributeList Attrs = CI->getAttributes(); - SmallVector ArgAttrs; - if (!Attrs.isEmpty() || !VarArgsAttrs.empty()) { - for (unsigned ArgNo = 0; - ArgNo < CI->getFunctionType()->getNumParams(); ++ArgNo) - ArgAttrs.push_back(Attrs.getParamAttributes(ArgNo)); - } - - // Add VarArg attributes. - ArgAttrs.append(VarArgsAttrs.begin(), VarArgsAttrs.end()); - Attrs = AttributeList::get(CI->getContext(), Attrs.getFnAttributes(), - Attrs.getRetAttributes(), ArgAttrs); - // Add VarArgs to existing parameters. - SmallVector Params(CI->arg_operands()); - Params.append(VarArgsToForward.begin(), VarArgsToForward.end()); - CallInst *NewCI = CallInst::Create( - CI->getFunctionType(), CI->getCalledOperand(), Params, "", CI); - NewCI->setDebugLoc(CI->getDebugLoc()); - NewCI->setAttributes(Attrs); - NewCI->setCallingConv(CI->getCallingConv()); + CallInst *NewCI = cast(forwardVAArgsToCall(CI)); CI->replaceAllUsesWith(NewCI); CI->eraseFromParent(); CI = NewCI; Index: llvm/test/Transforms/Inline/inline-va-arg-pack.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/Inline/inline-va-arg-pack.ll @@ -0,0 +1,80 @@ +; RUN: opt -inline -S < %s | FileCheck %s + +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.14.0" + +define available_externally i32 @wrapper(i32 %p, ...) { +entry: + %retval = call i32 (i32, ...) @vaargs(i32 %p) inline_va_arg_pack + ret i32 %retval +} + +define i32 @call_it() { +entry: + %val = call i32 (i32, ...) @wrapper(i32 1, i32 2, i32 3, i32 4) + ret i32 %val +} + +; CHECK-LABEL: define i32 @call_it +; CHECK-NEXT: entry: +; CHECK-NEXT: call i32 (i32, ...) @vaargs(i32 1, i32 2, i32 3, i32 4) +; CHECK-NEXT: ret i32 + +define i32 @call_it_many() { +entry: + %val = call i32 (i32, ...) @wrapper(i32 1, i8* null) + %val1 = call i32 (i32, ...) @wrapper(i32 2, i32 42) + ret i32 0 +} + +; CHECK-LABEL: define i32 @call_it_many +; CHECK-NEXT: entry: +; CHECK-NEXT: call i32 (i32, ...) @vaargs(i32 1, i8* null) +; CHECK-NEXT: call i32 (i32, ...) @vaargs(i32 2, i32 42) +; CHECK-NEXT: ret + +define available_externally i32 @wrapper2(...) { +entry: + %val1 = call i32 (...) @all_vaargs() inline_va_arg_pack + %val2 = call i32 (...) @all_vaargs() inline_va_arg_pack + %val3 = call i32 (...) @all_vaargs() + ret i32 0 +} + +define i32 @call_wrapper2() { +entry: + %val = call i32 (...) @wrapper2(i32 1, i32 2, i32 3) + ret i32 %val +} + +; CHECK-LABEL: define i32 @call_wrapper2 +; CHECK-NEXT: entry: +; CHECK-NEXT: call i32 (...) @all_vaargs(i32 1, i32 2, i32 3) +; CHECK-NEXT: call i32 (...) @all_vaargs(i32 1, i32 2, i32 3) +; CHECK-NEXT: call i32 (...) @all_vaargs() +; CHECK-NEXT: ret + +define available_externally i32 @uses_invoke(...) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + %val = invoke i32 (...) @all_vaargs() inline_va_arg_pack + to label %normal unwind label %lpad +normal: + ret i32 1 + +lpad: + %l = landingpad { i8*, i32 } cleanup + ret i32 2 +} + +define i32 @call_invoke() { +entry: + %val = call i32 (...) @uses_invoke(i32 1, i32 2, i32 3) + ret i32 %val +} + +; CHECK-LABEL: define i32 @call_invoke +; CHECK: invoke i32 (...) @all_vaargs(i32 1, i32 2, i32 3) + +declare i32 @vaargs(i32, ...) +declare i32 @all_vaargs(...) +declare i32 @__gxx_personality_v0(...) Index: llvm/test/Transforms/Inline/va-arg-pack-len.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/Inline/va-arg-pack-len.ll @@ -0,0 +1,58 @@ +; RUN: opt -inline -S < %s | FileCheck %s --dump-input-on-failure + +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.14.0" + +define available_externally i32 @inline_me(i32 %p, ...) { +entry: + %val = call i32 @llvm.va_arg_pack_len() + ret i32 %val +} + +define i32 @call_it() { +entry: + %val = call i32 (i32, ...) @inline_me(i32 1, i32 2, i32 3, i32 4) + ret i32 %val +} + +; CHECK-LABEL: define i32 @call_it +; CHECK-NEXT: entry: +; CHECK-NEXT: ret i32 3 + +define i32 @noargs() { +entry: + %val = call i32 (i32, ...) @inline_me(i32 1) + ret i32 %val +} + +; CHECK-LABEL: define i32 @noargs +; CHECK-NEXT: entry: +; CHECK-NEXT: ret i32 0 + +define available_externally i32 @inline_me2(...) { +entry: + %val = call i32 @llvm.va_arg_pack_len() + ret i32 %val +} + +define i32 @call_it2() { +entry: + %val = call i32 (...) @inline_me2(i32 1, i32 2, i32 3, i32 4) + ret i32 %val +} + +; CHECK-LABEL: define i32 @call_it2 +; CHECK-NEXT: entry: +; CHECK-NEXT: ret i32 4 + +define i32 @noargs2() { +entry: + %val = call i32 (...) @inline_me2() + ret i32 %val +} + +; CHECK-LABEL: define i32 @noargs +; CHECK-NEXT: entry: +; CHECK-NEXT: ret i32 0 + +declare i32 @llvm.va_arg_pack_len() Index: llvm/test/Verifier/va-arg-pack.ll =================================================================== --- /dev/null +++ llvm/test/Verifier/va-arg-pack.ll @@ -0,0 +1,28 @@ +; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s --dump-input-on-failure + +define void @bad_use1(...) { + call i32 () @llvm.va_arg_pack_len() + call void (...) @vaargs() inline_va_arg_pack + ret void + ; CHECK: call to @llvm.va_arg_pack_len can only appear in a function withavailable_externally linkage + ; CHECK: inline_va_arg_pack must be used in an available_externally function +} + +define available_externally void @bad_use2() { + call i32 () @llvm.va_arg_pack_len() + call void (...) @vaargs() inline_va_arg_pack + ; CHECK: call to @llvm.va_arg_pack_len can only appear in a variadic function + ; CHECK: inline_va_arg_pack can only appear in a variadic function + ret void +} + +define available_externally void @good_use(...) { + call i32 () @llvm.va_arg_pack_len() + call void (...) @vaargs() inline_va_arg_pack + ; CHECK-NOT: @llvm.va_arg_pack_len + ; CHECK-NOT: inline_va_arg_pack + ret void +} + +declare i32 @llvm.va_arg_pack_len() +declare void @vaargs(...)