Index: include/clang/Basic/Builtins.def =================================================================== --- include/clang/Basic/Builtins.def +++ include/clang/Basic/Builtins.def @@ -544,6 +544,10 @@ BUILTIN(__builtin_alloca_with_align, "v*zIz", "Fn") BUILTIN(__builtin_call_with_static_chain, "v.", "nt") +// Clang builtins (not available in GCC). +BUILTIN(__builtin_always_inline, "v.", "nt") +BUILTIN(__builtin_no_inline, "v.", "nt") + // "Overloaded" Atomic operator builtins. These are overloaded to support data // types of i8, i16, i32, i64, and i128. The front-end sees calls to the // non-suffixed version of these (which has a bogus type) and transforms them to Index: include/clang/Basic/DiagnosticSemaKinds.td =================================================================== --- include/clang/Basic/DiagnosticSemaKinds.td +++ include/clang/Basic/DiagnosticSemaKinds.td @@ -8230,6 +8230,15 @@ def err_second_argument_to_cwsc_not_pointer : Error< "second argument to __builtin_call_with_static_chain must be of pointer type">; +def err_argument_to_inline_intrinsic_not_call : Error< + "argument to %0 must be a function, member function, or operator call">; +def err_argument_to_inline_intrinsic_builtin_call : Error< + "argument to %0 must not be a builtin call">; +def err_argument_to_inline_intrinsic_pdtor_call : Error< + "argument to %0 must not be a pseudo-destructor call">; +def err_argument_to_inline_intrinsic_block_call : Error< + "argument to %0 must not be a block call">; + def err_vector_incorrect_num_initializers : Error< "%select{too many|too few}0 elements in vector initialization (expected %1 elements, have %2)">; def err_altivec_empty_initializer : Error<"expected initializer">; Index: include/clang/CodeGen/CGFunctionInfo.h =================================================================== --- include/clang/CodeGen/CGFunctionInfo.h +++ include/clang/CodeGen/CGFunctionInfo.h @@ -514,6 +514,8 @@ /// Whether this is a chain call. unsigned ChainCall : 1; + unsigned InlineCall : 2; + /// Whether this function is noreturn. unsigned NoReturn : 1; @@ -557,14 +559,17 @@ CGFunctionInfo() : Required(RequiredArgs::All) {} public: - static CGFunctionInfo *create(unsigned llvmCC, - bool instanceMethod, - bool chainCall, - const FunctionType::ExtInfo &extInfo, - ArrayRef paramInfos, - CanQualType resultType, - ArrayRef argTypes, - RequiredArgs required); + enum class CallInlineKind : unsigned char { + DefaultInline = 0, + NoInline = 1, + AlwaysInline = 2 + }; + static CGFunctionInfo * + create(unsigned llvmCC, bool instanceMethod, bool chainCall, + const FunctionType::ExtInfo &extInfo, + ArrayRef paramInfos, CanQualType resultType, + ArrayRef argTypes, RequiredArgs required, + CallInlineKind CIK = CallInlineKind::DefaultInline); void operator delete(void *p) { ::operator delete(p); } // Friending class TrailingObjects is apparently not good enough for MSVC, @@ -604,6 +609,9 @@ bool isInstanceMethod() const { return InstanceMethod; } bool isChainCall() const { return ChainCall; } + CallInlineKind getInlineCallKind() const { + return static_cast(InlineCall); + } bool isNoReturn() const { return NoReturn; } @@ -672,11 +680,15 @@ ArgStruct = Ty; ArgStructAlign = Align.getQuantity(); } + void setInlineKind(CallInlineKind CIK) { + InlineCall = static_cast(CIK); + } void Profile(llvm::FoldingSetNodeID &ID) { ID.AddInteger(getASTCallingConvention()); ID.AddBoolean(InstanceMethod); ID.AddBoolean(ChainCall); + ID.AddInteger(InlineCall); ID.AddBoolean(NoReturn); ID.AddBoolean(ReturnsRetained); ID.AddBoolean(NoCallerSavedRegs); @@ -693,17 +705,16 @@ for (const auto &I : arguments()) I.type.Profile(ID); } - static void Profile(llvm::FoldingSetNodeID &ID, - bool InstanceMethod, - bool ChainCall, - const FunctionType::ExtInfo &info, + static void Profile(llvm::FoldingSetNodeID &ID, bool InstanceMethod, + bool ChainCall, const FunctionType::ExtInfo &info, ArrayRef paramInfos, - RequiredArgs required, - CanQualType resultType, - ArrayRef argTypes) { + RequiredArgs required, CanQualType resultType, + ArrayRef argTypes, + CallInlineKind CIK = CallInlineKind::DefaultInline) { ID.AddInteger(info.getCC()); ID.AddBoolean(InstanceMethod); ID.AddBoolean(ChainCall); + ID.AddInteger(static_cast(CIK)); ID.AddBoolean(info.getNoReturn()); ID.AddBoolean(info.getProducesResult()); ID.AddBoolean(info.getNoCallerSavedRegs()); Index: lib/CodeGen/CGBuiltin.cpp =================================================================== --- lib/CodeGen/CGBuiltin.cpp +++ lib/CodeGen/CGBuiltin.cpp @@ -2993,6 +2993,30 @@ EmitCallee(Call->getCallee()), Call, ReturnValue, EmitScalarExpr(Chain)); } + + case Builtin::BI__builtin_no_inline: + case Builtin::BI__builtin_always_inline: { + CGFunctionInfo::CallInlineKind CIK = + BuiltinID == Builtin::BI__builtin_no_inline + ? CGFunctionInfo::CallInlineKind::NoInline + : CGFunctionInfo::CallInlineKind::AlwaysInline; + + auto *Arg = E->getArg(0); + if (const CXXMemberCallExpr *MemberCall = dyn_cast(Arg)) + return EmitCXXMemberCallExpr(MemberCall, ReturnValue, CIK); + + if (const CXXOperatorCallExpr *OperatorCall = + dyn_cast(Arg)) + if (const CXXMethodDecl *MD = + dyn_cast_or_null(OperatorCall->getCalleeDecl())) + return EmitCXXOperatorMemberCallExpr(OperatorCall, MD, ReturnValue, + CIK); + + const CallExpr *Call = cast(Arg); + return EmitCall(Call->getCallee()->getType(), EmitCallee(Call->getCallee()), + Call, ReturnValue, nullptr, CIK); + } + case Builtin::BI_InterlockedExchange8: case Builtin::BI_InterlockedExchange16: case Builtin::BI_InterlockedExchange: Index: lib/CodeGen/CGCall.cpp =================================================================== --- lib/CodeGen/CGCall.cpp +++ lib/CodeGen/CGCall.cpp @@ -559,12 +559,11 @@ /// Arrange a call as unto a free function, except possibly with an /// additional number of formal parameters considered required. static const CGFunctionInfo & -arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, - CodeGenModule &CGM, - const CallArgList &args, - const FunctionType *fnType, - unsigned numExtraRequiredArgs, - bool chainCall) { +arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, CodeGenModule &CGM, + const CallArgList &args, const FunctionType *fnType, + unsigned numExtraRequiredArgs, bool chainCall, + CGFunctionInfo::CallInlineKind CIK = + CGFunctionInfo::CallInlineKind::DefaultInline) { assert(args.size() >= numExtraRequiredArgs); llvm::SmallVector paramInfos; @@ -599,19 +598,18 @@ return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()), /*instanceMethod=*/false, chainCall, argTypes, fnType->getExtInfo(), paramInfos, - required); + required, CIK); } /// Figure out the rules for calling a function with the given formal /// type using the given arguments. The arguments are necessary /// because the function might be unprototyped, in which case it's /// target-dependent in crazy ways. -const CGFunctionInfo & -CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args, - const FunctionType *fnType, - bool chainCall) { +const CGFunctionInfo &CodeGenTypes::arrangeFreeFunctionCall( + const CallArgList &args, const FunctionType *fnType, bool chainCall, + CGFunctionInfo::CallInlineKind CIK) { return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, - chainCall ? 1 : 0, chainCall); + chainCall ? 1 : 0, chainCall, CIK); } /// A block function is essentially a free function with an @@ -671,11 +669,10 @@ /// /// numPrefixArgs is the number of ABI-specific prefix arguments we have. It /// does not count `this`. -const CGFunctionInfo & -CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args, - const FunctionProtoType *proto, - RequiredArgs required, - unsigned numPrefixArgs) { +const CGFunctionInfo &CodeGenTypes::arrangeCXXMethodCall( + const CallArgList &args, const FunctionProtoType *proto, + RequiredArgs required, unsigned numPrefixArgs, + CGFunctionInfo::CallInlineKind CIK) { assert(numPrefixArgs + 1 <= args.size() && "Emitting a call with less args than the required prefix?"); // Add one to account for `this`. It's a bit awkward here, but we don't count @@ -689,7 +686,7 @@ FunctionType::ExtInfo info = proto->getExtInfo(); return arrangeLLVMFunctionInfo( GetReturnType(proto->getReturnType()), /*instanceMethod=*/true, - /*chainCall=*/false, argTypes, info, paramInfos, required); + /*chainCall=*/false, argTypes, info, paramInfos, required, CIK); } const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() { @@ -733,21 +730,18 @@ /// Arrange the argument and result information for an abstract value /// of a given function type. This is the method which all of the /// above functions ultimately defer to. -const CGFunctionInfo & -CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType, - bool instanceMethod, - bool chainCall, - ArrayRef argTypes, - FunctionType::ExtInfo info, - ArrayRef paramInfos, - RequiredArgs required) { +const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo( + CanQualType resultType, bool instanceMethod, bool chainCall, + ArrayRef argTypes, FunctionType::ExtInfo info, + ArrayRef paramInfos, + RequiredArgs required, CGFunctionInfo::CallInlineKind CIK) { assert(std::all_of(argTypes.begin(), argTypes.end(), [](CanQualType T) { return T.isCanonicalAsParam(); })); // Lookup or create unique function info. llvm::FoldingSetNodeID ID; CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos, - required, resultType, argTypes); + required, resultType, argTypes, CIK); void *insertPos = nullptr; CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos); @@ -757,8 +751,8 @@ unsigned CC = ClangCallConvToLLVMCallConv(info.getCC()); // Construct the function info. We co-allocate the ArgInfos. - FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info, - paramInfos, resultType, argTypes, required); + FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info, paramInfos, + resultType, argTypes, required, CIK); FunctionInfos.InsertNode(FI, insertPos); bool inserted = FunctionsBeingProcessed.insert(FI).second; @@ -793,14 +787,12 @@ return *FI; } -CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, - bool instanceMethod, - bool chainCall, - const FunctionType::ExtInfo &info, - ArrayRef paramInfos, - CanQualType resultType, - ArrayRef argTypes, - RequiredArgs required) { +CGFunctionInfo * +CGFunctionInfo::create(unsigned llvmCC, bool instanceMethod, bool chainCall, + const FunctionType::ExtInfo &info, + ArrayRef paramInfos, + CanQualType resultType, ArrayRef argTypes, + RequiredArgs required, CallInlineKind CIK) { assert(paramInfos.empty() || paramInfos.size() == argTypes.size()); void *buffer = @@ -813,6 +805,7 @@ FI->ASTCallingConvention = info.getCC(); FI->InstanceMethod = instanceMethod; FI->ChainCall = chainCall; + FI->InlineCall = static_cast(CIK); FI->NoReturn = info.getNoReturn(); FI->ReturnsRetained = info.getProducesResult(); FI->NoCallerSavedRegs = info.getNoCallerSavedRegs(); @@ -2005,6 +1998,11 @@ llvm::AttributeSet::get(getLLVMContext(), Attrs); } + if (FI.getInlineCallKind() == CGFunctionInfo::CallInlineKind::AlwaysInline) + FuncAttrs.addAttribute(llvm::Attribute::AlwaysInline); + else if (FI.getInlineCallKind() == CGFunctionInfo::CallInlineKind::NoInline) + FuncAttrs.addAttribute(llvm::Attribute::NoInline); + unsigned ArgNo = 0; for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(), E = FI.arg_end(); @@ -3784,7 +3782,8 @@ ReturnValueSlot ReturnValue, const CallArgList &CallArgs, llvm::Instruction **callOrInvoke, - SourceLocation Loc) { + SourceLocation Loc, + CGFunctionInfo::CallInlineKind CIK) { // FIXME: We no longer need the types from CallArgs; lift up and simplify. assert(Callee.isOrdinary() || Callee.isVirtual()); @@ -4255,6 +4254,13 @@ llvm::Attribute::AlwaysInline); } + if (CIK != CGFunctionInfo::CallInlineKind::DefaultInline) + Attrs = + Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex, + CIK == CGFunctionInfo::CallInlineKind::NoInline + ? llvm::Attribute::NoInline + : llvm::Attribute::AlwaysInline); + // Disable inlining inside SEH __try blocks. if (isSEHTryScope()) { Attrs = Index: lib/CodeGen/CGExpr.cpp =================================================================== --- lib/CodeGen/CGExpr.cpp +++ lib/CodeGen/CGExpr.cpp @@ -4585,9 +4585,11 @@ AlignmentSource::Decl); } -RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee, - const CallExpr *E, ReturnValueSlot ReturnValue, - llvm::Value *Chain) { +RValue CodeGenFunction::EmitCall(QualType CalleeType, + const CGCallee &OrigCallee, const CallExpr *E, + ReturnValueSlot ReturnValue, + llvm::Value *Chain, + CGFunctionInfo::CallInlineKind CIK) { // Get the actual function type. The callee type will always be a pointer to // function type or a block pointer type. assert(CalleeType->isFunctionPointerType() && @@ -4732,8 +4734,8 @@ EmitCallArgs(Args, dyn_cast(FnType), E->arguments(), E->getDirectCallee(), /*ParamsToSkip*/ 0, Order); - const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall( - Args, FnType, /*isChainCall=*/Chain); + const CGFunctionInfo *FnInfo = &CGM.getTypes().arrangeFreeFunctionCall( + Args, FnType, /*isChainCall=*/Chain, CIK); // C99 6.5.2.2p6: // If the expression that denotes the called function has a type @@ -4756,7 +4758,7 @@ // Chain calls use this same code path to add the invisible chain parameter // to the function type. if (isa(FnType) || Chain) { - llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo); + llvm::Type *CalleeTy = getTypes().GetFunctionType(*FnInfo); CalleeTy = CalleeTy->getPointerTo(); llvm::Value *CalleePtr = Callee.getFunctionPointer(); @@ -4764,7 +4766,7 @@ Callee.setFunctionPointer(CalleePtr); } - return EmitCall(FnInfo, Callee, ReturnValue, Args, nullptr, E->getExprLoc()); + return EmitCall(*FnInfo, Callee, ReturnValue, Args, nullptr, E->getExprLoc()); } LValue CodeGenFunction:: Index: lib/CodeGen/CGExprCXX.cpp =================================================================== --- lib/CodeGen/CGExprCXX.cpp +++ lib/CodeGen/CGExprCXX.cpp @@ -80,15 +80,15 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorCall( const CXXMethodDecl *MD, const CGCallee &Callee, - ReturnValueSlot ReturnValue, - llvm::Value *This, llvm::Value *ImplicitParam, QualType ImplicitParamTy, - const CallExpr *CE, CallArgList *RtlArgs) { + ReturnValueSlot ReturnValue, llvm::Value *This, llvm::Value *ImplicitParam, + QualType ImplicitParamTy, const CallExpr *CE, CallArgList *RtlArgs, + CGFunctionInfo::CallInlineKind CIK) { const FunctionProtoType *FPT = MD->getType()->castAs(); CallArgList Args; MemberCallInfo CallInfo = commonEmitCXXMemberOrOperatorCall( *this, MD, This, ImplicitParam, ImplicitParamTy, CE, Args, RtlArgs); auto &FnInfo = CGM.getTypes().arrangeCXXMethodCall( - Args, FPT, CallInfo.ReqArgs, CallInfo.PrefixSize); + Args, FPT, CallInfo.ReqArgs, CallInfo.PrefixSize, CIK); return EmitCall(FnInfo, Callee, ReturnValue, Args, nullptr, CE ? CE->getExprLoc() : SourceLocation()); } @@ -165,12 +165,16 @@ // Note: This function also emit constructor calls to support a MSVC // extensions allowing explicit constructor function call. -RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE, - ReturnValueSlot ReturnValue) { +RValue +CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE, + ReturnValueSlot ReturnValue, + CGFunctionInfo::CallInlineKind CIK) { const Expr *callee = CE->getCallee()->IgnoreParens(); - if (isa(callee)) - return EmitCXXMemberPointerCallExpr(CE, ReturnValue); + if (isa(callee)) { + llvm::errs() << "lolol\n"; + return EmitCXXMemberPointerCallExpr(CE, ReturnValue, CIK); + } const MemberExpr *ME = cast(callee); const CXXMethodDecl *MD = cast(ME->getMemberDecl()); @@ -179,7 +183,7 @@ // The method is static, emit it as we would a regular call. CGCallee callee = CGCallee::forDirect(CGM.GetAddrOfFunction(MD), MD); return EmitCall(getContext().getPointerType(MD->getType()), callee, CE, - ReturnValue); + ReturnValue, /*Chain = */ nullptr, CIK); } bool HasQualifier = ME->hasQualifier(); @@ -188,13 +192,13 @@ const Expr *Base = ME->getBase(); return EmitCXXMemberOrOperatorMemberCallExpr( - CE, MD, ReturnValue, HasQualifier, Qualifier, IsArrow, Base); + CE, MD, ReturnValue, HasQualifier, Qualifier, IsArrow, Base, CIK); } RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue, bool HasQualifier, NestedNameSpecifier *Qualifier, bool IsArrow, - const Expr *Base) { + const Expr *Base, CGFunctionInfo::CallInlineKind CIK) { assert(isa(CE) || isa(CE)); // Compute the object pointer. @@ -406,12 +410,12 @@ return EmitCXXMemberOrOperatorCall( CalleeDecl, Callee, ReturnValue, This.getPointer(), - /*ImplicitParam=*/nullptr, QualType(), CE, RtlArgs); + /*ImplicitParam=*/nullptr, QualType(), CE, RtlArgs, CIK); } -RValue -CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E, - ReturnValueSlot ReturnValue) { +RValue CodeGenFunction::EmitCXXMemberPointerCallExpr( + const CXXMemberCallExpr *E, ReturnValueSlot ReturnValue, + CGFunctionInfo::CallInlineKind CIK) { const BinaryOperator *BO = cast(E->getCallee()->IgnoreParens()); const Expr *BaseExpr = BO->getLHS(); @@ -459,18 +463,17 @@ EmitCallArgs(Args, FPT, E->arguments()); return EmitCall(CGM.getTypes().arrangeCXXMethodCall(Args, FPT, required, /*PrefixSize=*/0), - Callee, ReturnValue, Args, nullptr, E->getExprLoc()); + Callee, ReturnValue, Args, nullptr, E->getExprLoc(), CIK); } -RValue -CodeGenFunction::EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E, - const CXXMethodDecl *MD, - ReturnValueSlot ReturnValue) { +RValue CodeGenFunction::EmitCXXOperatorMemberCallExpr( + const CXXOperatorCallExpr *E, const CXXMethodDecl *MD, + ReturnValueSlot ReturnValue, CGFunctionInfo::CallInlineKind CIK) { assert(MD->isInstance() && "Trying to emit a member call expr on a static method!"); return EmitCXXMemberOrOperatorMemberCallExpr( E, MD, ReturnValue, /*HasQualifier=*/false, /*Qualifier=*/nullptr, - /*IsArrow=*/false, E->getArg(0)); + /*IsArrow=*/false, E->getArg(0), CIK); } RValue CodeGenFunction::EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E, Index: lib/CodeGen/CodeGenFunction.h =================================================================== --- lib/CodeGen/CodeGenFunction.h +++ lib/CodeGen/CodeGenFunction.h @@ -3565,15 +3565,21 @@ /// LLVM arguments and the types they were derived from. RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, - llvm::Instruction **callOrInvoke, SourceLocation Loc); + llvm::Instruction **callOrInvoke, SourceLocation Loc, + CGFunctionInfo::CallInlineKind CIK = + CGFunctionInfo::CallInlineKind::DefaultInline); RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, - llvm::Instruction **callOrInvoke = nullptr) { + llvm::Instruction **callOrInvoke = nullptr, + CGFunctionInfo::CallInlineKind CIK = + CGFunctionInfo::CallInlineKind::DefaultInline) { return EmitCall(CallInfo, Callee, ReturnValue, Args, callOrInvoke, - SourceLocation()); + SourceLocation(), CIK); } RValue EmitCall(QualType FnType, const CGCallee &Callee, const CallExpr *E, - ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr); + ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr, + CGFunctionInfo::CallInlineKind CIK = + CGFunctionInfo::CallInlineKind::DefaultInline); RValue EmitCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue = ReturnValueSlot()); RValue EmitSimpleCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue); @@ -3637,38 +3643,44 @@ void callCStructCopyAssignmentOperator(LValue Dst, LValue Src); void callCStructMoveAssignmentOperator(LValue Dst, LValue Src); - RValue - EmitCXXMemberOrOperatorCall(const CXXMethodDecl *Method, - const CGCallee &Callee, - ReturnValueSlot ReturnValue, llvm::Value *This, - llvm::Value *ImplicitParam, - QualType ImplicitParamTy, const CallExpr *E, - CallArgList *RtlArgs); + RValue EmitCXXMemberOrOperatorCall( + const CXXMethodDecl *Method, const CGCallee &Callee, + ReturnValueSlot ReturnValue, llvm::Value *This, + llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *E, + CallArgList *RtlArgs, + CGFunctionInfo::CallInlineKind CIK = + CGFunctionInfo::CallInlineKind::DefaultInline); RValue EmitCXXDestructorCall(const CXXDestructorDecl *DD, const CGCallee &Callee, llvm::Value *This, llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *E, StructorType Type); - RValue EmitCXXMemberCallExpr(const CXXMemberCallExpr *E, - ReturnValueSlot ReturnValue); - RValue EmitCXXMemberOrOperatorMemberCallExpr(const CallExpr *CE, - const CXXMethodDecl *MD, - ReturnValueSlot ReturnValue, - bool HasQualifier, - NestedNameSpecifier *Qualifier, - bool IsArrow, const Expr *Base); + RValue + EmitCXXMemberCallExpr(const CXXMemberCallExpr *E, ReturnValueSlot ReturnValue, + CGFunctionInfo::CallInlineKind CIK = + CGFunctionInfo::CallInlineKind::DefaultInline); + RValue EmitCXXMemberOrOperatorMemberCallExpr( + const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue, + bool HasQualifier, NestedNameSpecifier *Qualifier, bool IsArrow, + const Expr *Base, + CGFunctionInfo::CallInlineKind CIK = + CGFunctionInfo::CallInlineKind::DefaultInline); // Compute the object pointer. Address EmitCXXMemberDataPointerAddress(const Expr *E, Address base, llvm::Value *memberPtr, const MemberPointerType *memberPtrType, LValueBaseInfo *BaseInfo = nullptr, TBAAAccessInfo *TBAAInfo = nullptr); - RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E, - ReturnValueSlot ReturnValue); - - RValue EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E, - const CXXMethodDecl *MD, - ReturnValueSlot ReturnValue); + RValue EmitCXXMemberPointerCallExpr( + const CXXMemberCallExpr *E, ReturnValueSlot ReturnValue, + CGFunctionInfo::CallInlineKind CIK = + CGFunctionInfo::CallInlineKind::DefaultInline); + + RValue EmitCXXOperatorMemberCallExpr( + const CXXOperatorCallExpr *E, const CXXMethodDecl *MD, + ReturnValueSlot ReturnValue, + CGFunctionInfo::CallInlineKind CIK = + CGFunctionInfo::CallInlineKind::DefaultInline); RValue EmitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E); RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E, Index: lib/CodeGen/CodeGenTypes.h =================================================================== --- lib/CodeGen/CodeGenTypes.h +++ lib/CodeGen/CodeGenTypes.h @@ -261,9 +261,11 @@ /// Free functions are functions that are compatible with an ordinary /// C function pointer type. const CGFunctionInfo &arrangeFunctionDeclaration(const FunctionDecl *FD); - const CGFunctionInfo &arrangeFreeFunctionCall(const CallArgList &Args, - const FunctionType *Ty, - bool ChainCall); + const CGFunctionInfo & + arrangeFreeFunctionCall(const CallArgList &Args, const FunctionType *Ty, + bool ChainCall, + CGFunctionInfo::CallInlineKind CIK = + CGFunctionInfo::CallInlineKind::DefaultInline); const CGFunctionInfo &arrangeFreeFunctionType(CanQual Ty, const FunctionDecl *FD); const CGFunctionInfo &arrangeFreeFunctionType(CanQual Ty); @@ -309,10 +311,11 @@ unsigned ExtraSuffixArgs, bool PassProtoArgs = true); - const CGFunctionInfo &arrangeCXXMethodCall(const CallArgList &args, - const FunctionProtoType *type, - RequiredArgs required, - unsigned numPrefixArgs); + const CGFunctionInfo & + arrangeCXXMethodCall(const CallArgList &args, const FunctionProtoType *type, + RequiredArgs required, unsigned numPrefixArgs, + CGFunctionInfo::CallInlineKind CIK = + CGFunctionInfo::CallInlineKind::DefaultInline); const CGFunctionInfo & arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD); const CGFunctionInfo &arrangeMSCtorClosure(const CXXConstructorDecl *CD, @@ -327,13 +330,13 @@ /// this. /// /// \param argTypes - must all actually be canonical as params - const CGFunctionInfo &arrangeLLVMFunctionInfo(CanQualType returnType, - bool instanceMethod, - bool chainCall, - ArrayRef argTypes, - FunctionType::ExtInfo info, - ArrayRef paramInfos, - RequiredArgs args); + const CGFunctionInfo &arrangeLLVMFunctionInfo( + CanQualType returnType, bool instanceMethod, bool chainCall, + ArrayRef argTypes, FunctionType::ExtInfo info, + ArrayRef paramInfos, + RequiredArgs args, + CGFunctionInfo::CallInlineKind CIK = + CGFunctionInfo::CallInlineKind::DefaultInline); /// Compute a new LLVM record layout object for the given record. CGRecordLayout *ComputeRecordLayout(const RecordDecl *D, Index: lib/Sema/SemaChecking.cpp =================================================================== --- lib/Sema/SemaChecking.cpp +++ lib/Sema/SemaChecking.cpp @@ -327,6 +327,61 @@ return false; } +static bool SemaBuiltinCallWithInline(Sema &S, CallExpr *BuiltinCall) { + if (checkArgCount(S, BuiltinCall, 1)) + return true; + + SourceLocation BuiltinLoc = BuiltinCall->getBeginLoc(); + Expr *Builtin = BuiltinCall->getCallee()->IgnoreImpCasts(); + Expr *Call = BuiltinCall->getArg(0); + + const auto CallStmtClass = Call->getStmtClass(); + if (CallStmtClass != Stmt::CallExprClass && + CallStmtClass != Stmt::CXXMemberCallExprClass && + CallStmtClass != Stmt::CXXOperatorCallExprClass) { + S.Diag(BuiltinLoc, diag::err_argument_to_inline_intrinsic_not_call) + << Builtin << Call->getSourceRange(); + return true; + } + + auto *CE = cast(Call); + const Decl *TargetDecl = CE->getCalleeDecl(); + if (const auto *FD = dyn_cast_or_null(TargetDecl)) + if (FD->getBuiltinID()) { + S.Diag(BuiltinLoc, diag::err_argument_to_inline_intrinsic_builtin_call) + << Builtin << Call->getSourceRange(); + return true; + } + + if (CE->getCallee()->getType()->isBlockPointerType()) { + S.Diag(BuiltinLoc, diag::err_argument_to_inline_intrinsic_block_call) + << Builtin << Call->getSourceRange(); + return true; + } + + if (isa(CE->getCallee()->IgnoreParens())) { + S.Diag(BuiltinLoc, diag::err_argument_to_inline_intrinsic_pdtor_call) + << Builtin << Call->getSourceRange(); + return true; + } + + QualType ReturnTy = CE->getCallReturnType(S.Context); + QualType ArgTys[1] = {ReturnTy}; + QualType BuiltinTy = S.Context.getFunctionType( + ReturnTy, ArgTys, FunctionProtoType::ExtProtoInfo()); + QualType BuiltinPtrTy = S.Context.getPointerType(BuiltinTy); + + Builtin = + S.ImpCastExprToType(Builtin, BuiltinPtrTy, CK_BuiltinFnToFnPtr).get(); + + BuiltinCall->setType(CE->getType()); + BuiltinCall->setValueKind(CE->getValueKind()); + BuiltinCall->setObjectKind(CE->getObjectKind()); + BuiltinCall->setCallee(Builtin); + + return false; +} + static bool SemaBuiltinSEHScopeCheck(Sema &SemaRef, CallExpr *TheCall, Scope::ScopeFlags NeededScopeFlags, unsigned DiagID) { @@ -1239,6 +1294,11 @@ if (SemaBuiltinCallWithStaticChain(*this, TheCall)) return ExprError(); break; + case Builtin::BI__builtin_always_inline: + case Builtin::BI__builtin_no_inline: + if (SemaBuiltinCallWithInline(*this, TheCall)) + return ExprError(); + break; case Builtin::BI__exception_code: case Builtin::BI_exception_code: if (SemaBuiltinSEHScopeCheck(*this, TheCall, Scope::SEHExceptScope, Index: test/CodeGenCXX/inline-builtin-call.cpp =================================================================== --- /dev/null +++ test/CodeGenCXX/inline-builtin-call.cpp @@ -0,0 +1,127 @@ +// RUN: %clang_cc1 -std=c++17 -O0 -disable-O0-optnone -triple x86_64-unknown-linux -emit-llvm -o - %s | FileCheck %s + +struct S { + S() {} + void foo(); + void bar(int); + int baz(float, char); + static void lol(); + + virtual void virt(){}; + void operator++(); + friend S operator+(const S &, const S &); +}; + +void test_always() { + S s; + + // CHECK: call void @_ZN1S3fooEv({{.*}}) [[ATTR_ALWAYS:#[0-9]+]] + __builtin_always_inline(s.foo()); + + // CHECK-NOT: call void @_ZN1S3fooEv({{.*}}) [[ATTR_ALWAYS]] + // CHECK: call void @_ZN1S3fooEv({{.*}}) + s.foo(); + + // CHECK: call void @_ZN1S3barEi({{.*}}) [[ATTR_ALWAYS]] + __builtin_always_inline(s.bar(42)); + + // CHECK: call i32 @_ZN1S3bazEfc({{.*}}) [[ATTR_ALWAYS]] + __builtin_always_inline(s.baz(3.14f, 'x')); + + // CHECK: call i32 @_ZN1S3bazEfc({{.*}}) [[ATTR_ALWAYS]] + // CHECK-NOT: call void @_ZN1S3barEi({{.*}}) [[ATTR_ALWAYS]] + // CHECK: call void @_ZN1S3barEi({{.*}}) + s.bar(__builtin_always_inline(s.baz(3.14f, 'x'))); + + // CHECK-NOT: call i32 @_ZN1S3bazEfc({{.*}}) [[ATTR_ALWAYS]] + // CHECK: call i32 @_ZN1S3bazEfc({{.*}}) + // CHECK: call void @_ZN1S3barEi({{.*}}) [[ATTR_ALWAYS]] + __builtin_always_inline(s.bar(s.baz(3.14f, 'x'))); + + S s2; + + // CHECK-NOT: call void @_ZN1SppEv({{.*}}) [[ATTR_ALWAYS]] + // CHECK: call void @_ZN1SppEv({{.*}}) + ++s2; + // CHECK: call void @_ZN1SppEv({{.*}}) [[ATTR_ALWAYS]] + __builtin_always_inline(++s2); + + // CHECK: call void @_ZplRK1SS1_({{.*}}) [[ATTR_ALWAYS]] + __builtin_always_inline(s + s2); + + // CHECK: call void @_ZN1S3lolEv({{.*}}) [[ATTR_ALWAYS]] + __builtin_always_inline(s.lol()); + + // CHECK: call void @_ZN1S3lolEv({{.*}}) [[ATTR_ALWAYS]] + __builtin_always_inline(S::lol()); +} + +void test_no_inline() { + S s; + + // CHECK: call void @_ZN1S3fooEv({{.*}}) [[ATTR_NO:#[0-9]+]] + __builtin_no_inline(s.foo()); + + // CHECK-NOT: call void @_ZN1S3fooEv({{.*}}) [[ATTR_NO]] + // CHECK: call void @_ZN1S3fooEv({{.*}}) + s.foo(); + + // CHECK: call void @_ZN1S3barEi({{.*}}) [[ATTR_NO]] + __builtin_no_inline(s.bar(42)); + + // CHECK: call i32 @_ZN1S3bazEfc({{.*}}) [[ATTR_NO]] + __builtin_no_inline(s.baz(3.14f, 'x')); + + // CHECK: call i32 @_ZN1S3bazEfc({{.*}}) [[ATTR_NO]] + // CHECK-NOT: call void @_ZN1S3barEi({{.*}}) [[ATTR_NO]] + // CHECK: call void @_ZN1S3barEi({{.*}}) + s.bar(__builtin_no_inline(s.baz(3.14f, 'x'))); + + // CHECK-NOT: call i32 @_ZN1S3bazEfc({{.*}}) [[ATTR_NO]] + // CHECK: call i32 @_ZN1S3bazEfc({{.*}}) + // CHECK: call void @_ZN1S3barEi({{.*}}) [[ATTR_NO]] + __builtin_no_inline(s.bar(s.baz(3.14f, 'x'))); + + S s2; + + // CHECK-NOT: call void @_ZN1SppEv({{.*}}) [[ATTR_NO]] + // CHECK: call void @_ZN1SppEv({{.*}}) + ++s2; + // CHECK: call void @_ZN1SppEv({{.*}}) [[ATTR_NO]] + __builtin_no_inline(++s2); + + // CHECK: call void @_ZplRK1SS1_({{.*}}) [[ATTR_NO]] + __builtin_no_inline(s + s2); + + // CHECK: call void @_ZN1S3lolEv({{.*}}) [[ATTR_NO]] + __builtin_no_inline(s.lol()); + + // CHECK: call void @_ZN1S3lolEv({{.*}}) [[ATTR_NO]] + __builtin_no_inline(S::lol()); +} + +S &getS(); + +void test_indirect() { + auto fptr = &test_always; + // CHECK: call void %{{.*}} [[ATTR_ALWAYS]] + __builtin_always_inline(fptr()); + // CHECK: call void %{{.*}} [[ATTR_NO]] + __builtin_no_inline(fptr()); + + // CHECK: call dereferenceable(8) %struct.S* @_Z4getSv() + S &s = getS(); + // CHECK: call void %{{.*}} [[ATTR_ALWAYS]] + __builtin_always_inline(s.virt()); + // CHECK: call void %{{.*}} [[ATTR_NO]] + __builtin_no_inline(s.virt()); + + auto mptr = &S::foo; + // CHECK: call void %{{.*}} [[ATTR_ALWAYS]] + __builtin_always_inline((s.*mptr)()); + // CHECK: call void %{{.*}} [[ATTR_NO]] + __builtin_no_inline((s.*mptr)()); +} + +// CHECK: attributes [[ATTR_ALWAYS]] = { alwaysinline } +// CHECK: attributes [[ATTR_NO]] = { noinline } Index: test/Sema/inline-builtins.c =================================================================== --- /dev/null +++ test/Sema/inline-builtins.c @@ -0,0 +1,24 @@ +// RUN: %clang_cc1 -fsyntax-only -fblocks -verify %s + +void foo(); + +void test() { + void (*bar)() = &foo; + __builtin_always_inline(foo()); + __builtin_no_inline(foo()); + + __builtin_always_inline(bar()); + __builtin_no_inline(bar()); + __builtin_always_inline((*bar)()); + __builtin_no_inline((*bar)()); + + // clang-format off + __builtin_always_inline(^{}()); // expected-error {{argument to __builtin_always_inline must not be a block call}} + __builtin_no_inline(^{}()); // expected-error {{argument to __builtin_no_inline must not be a block call}} + // clang-format on + + __builtin_always_inline(__builtin_always_inline(foo())); // expected-error {{argument to __builtin_always_inline must not be a builtin call}} + __builtin_always_inline(__builtin_no_inline(foo())); // expected-error {{argument to __builtin_always_inline must not be a builtin call}} + __builtin_no_inline(__builtin_no_inline(foo())); // expected-error {{argument to __builtin_no_inline must not be a builtin call}} + __builtin_no_inline(__builtin_always_inline(foo())); // expected-error {{argument to __builtin_no_inline must not be a builtin call}} +} Index: test/SemaCXX/inline-builtins.cpp =================================================================== --- /dev/null +++ test/SemaCXX/inline-builtins.cpp @@ -0,0 +1,128 @@ +// RUN: %clang_cc1 -std=c++17 -fsyntax-only -verify %s + +struct S { + S(); + void foo(); + void bar(int); + int baz(float, char); + static void lol(); + + virtual void virt(); + void operator++(); + friend S operator+(const S &, const S &); +}; + +S operator"" _s(unsigned long long); + +S &getS(); + +void test_positive() { + S &s = getS(); + __builtin_always_inline(s.foo()); + __builtin_no_inline(s.foo()); + + __builtin_always_inline(getS().foo()); + __builtin_no_inline(getS().foo()); + + __builtin_always_inline(getS()).foo(); + __builtin_no_inline(getS()).foo(); + + __builtin_always_inline(s.bar(1)); + __builtin_no_inline(s.bar(1)); + + int a = __builtin_always_inline(s.baz(3.14, 'a')); + int b = __builtin_no_inline(s.baz(3.14, 'a')); + + __builtin_always_inline(s.bar(s.baz(3.14, 'a'))); + __builtin_no_inline(s.bar(s.baz(3.14, 'a'))); + + s.bar(__builtin_always_inline(s.baz(3.14, 'a'))); + s.bar(__builtin_no_inline(s.baz(3.14, 'a'))); + + __builtin_always_inline(s.lol()); + __builtin_no_inline(s.lol()); + + __builtin_always_inline(S::lol()); + __builtin_no_inline(S::lol()); + + __builtin_always_inline(s.virt()); + __builtin_no_inline(s.virt()); + + __builtin_always_inline(++s); + __builtin_no_inline(++s); + + __builtin_always_inline(s.operator++()); + __builtin_no_inline(s.operator++()); + + S s1; + __builtin_always_inline(s + s1); + __builtin_no_inline(s + s1); + + auto mptr = &S::foo; + __builtin_always_inline((s.*mptr)()); + __builtin_no_inline((s.*mptr)()); + + __builtin_always_inline(s.~S()); + __builtin_no_inline(s.~S()); + + __builtin_always_inline([] {}()); + __builtin_no_inline([] {}()); + + auto lam = [&a, b](S &ss) { return ++a + b; }; + int c = __builtin_always_inline(lam(s1)); + int d = __builtin_no_inline(lam(s1)); +} + +template +void callDtorAlways(T &t) { + __builtin_always_inline(t.~T()); // expected-error {{argument to __builtin_always_inline must not be a pseudo-destructor call}} +} +template +void callDtorNo(T &t) { + __builtin_no_inline(t.~T()); // expected-error {{argument to __builtin_no_inline must not be a pseudo-destructor call}} +} + +void test_errors() { + using I = int; + I e = 1; + __builtin_always_inline(e.~I()); // expected-error {{argument to __builtin_always_inline must not be a pseudo-destructor call}} + __builtin_no_inline(e.~I()); // expected-error {{argument to __builtin_no_inline must not be a pseudo-destructor call}} + + S s; + callDtorAlways(s); + callDtorNo(s); + + callDtorAlways(e); // expected-note {{in instantiation of function template specialization 'callDtorAlways' requested here}} + callDtorNo(e); // expected-note {{in instantiation of function template specialization 'callDtorNo' requested here}} + + // TODO: Support User Defined Literals. + S s2 = __builtin_always_inline(1_s); // expected-error {{argument to __builtin_always_inline must be a function, member function, or operator call}} + s2 = __builtin_no_inline(1_s); // expected-error {{argument to __builtin_no_inline must be a function, member function, or operator call}} + + // TODO: This should be handled as well. + S s3 = __builtin_always_inline(S()); // expected-error {{argument to __builtin_always_inline must be a function, member function, or operator call}} + S s4 = __builtin_no_inline(S()); // expected-error {{argument to __builtin_no_inline must be a function, member function, or operator call}} + S s5 = __builtin_always_inline(S{}); // expected-error {{argument to __builtin_always_inline must be a function, member function, or operator call}} + S s6 = __builtin_no_inline(S{}); // expected-error {{argument to __builtin_no_inline must be a function, member function, or operator call}} +} + +constexpr int f1() { return 1; } + +// TODO: It should be possible to make the inline intrinsics constexpr-friendly. +constexpr int f2(int x) { // expected-error {{constexpr function never produces a constant expression}} + return x + __builtin_always_inline(f1()); // expected-note {{subexpression not valid in a constant expression}} +} + +template +struct IsSame { static constexpr bool value = false; }; + +template +struct IsSame { static constexpr bool value = true; }; + +void test_unevaluated_context() { + static_assert(IsSame::value, ""); + static_assert(IsSame::value, ""); + + static_assert(!IsSame::value, ""); + static_assert(!IsSame::value, ""); +}