diff --git a/clang/include/clang/CodeGen/CGFunctionInfo.h b/clang/include/clang/CodeGen/CGFunctionInfo.h --- a/clang/include/clang/CodeGen/CGFunctionInfo.h +++ b/clang/include/clang/CodeGen/CGFunctionInfo.h @@ -567,6 +567,10 @@ /// Whether this is a chain call. unsigned ChainCall : 1; + /// Whether this function is called by forwarding arguments. + /// This doesn't support inalloca or varargs. + unsigned DelegateCall : 1; + /// Whether this function is a CMSE nonsecure call unsigned CmseNSCall : 1; @@ -616,14 +620,11 @@ CGFunctionInfo() : Required(RequiredArgs::All) {} public: - static CGFunctionInfo *create(unsigned llvmCC, - bool instanceMethod, - bool chainCall, - const FunctionType::ExtInfo &extInfo, - ArrayRef paramInfos, - CanQualType resultType, - ArrayRef argTypes, - RequiredArgs required); + static CGFunctionInfo * + create(unsigned llvmCC, bool instanceMethod, bool chainCall, + bool delegateCall, const FunctionType::ExtInfo &extInfo, + ArrayRef paramInfos, CanQualType resultType, + ArrayRef argTypes, RequiredArgs required); void operator delete(void *p) { ::operator delete(p); } // Friending class TrailingObjects is apparently not good enough for MSVC, @@ -663,6 +664,8 @@ bool isChainCall() const { return ChainCall; } + bool isDelegateCall() const { return DelegateCall; } + bool isCmseNSCall() const { return CmseNSCall; } bool isNoReturn() const { return NoReturn; } @@ -749,6 +752,7 @@ ID.AddInteger(getASTCallingConvention()); ID.AddBoolean(InstanceMethod); ID.AddBoolean(ChainCall); + ID.AddBoolean(DelegateCall); ID.AddBoolean(NoReturn); ID.AddBoolean(ReturnsRetained); ID.AddBoolean(NoCallerSavedRegs); @@ -766,17 +770,16 @@ for (const auto &I : arguments()) I.type.Profile(ID); } - static void Profile(llvm::FoldingSetNodeID &ID, - bool InstanceMethod, - bool ChainCall, + static void Profile(llvm::FoldingSetNodeID &ID, bool InstanceMethod, + bool ChainCall, bool IsDelegateCall, const FunctionType::ExtInfo &info, ArrayRef paramInfos, - RequiredArgs required, - CanQualType resultType, + RequiredArgs required, CanQualType resultType, ArrayRef argTypes) { ID.AddInteger(info.getCC()); ID.AddBoolean(InstanceMethod); ID.AddBoolean(ChainCall); + ID.AddBoolean(IsDelegateCall); ID.AddBoolean(info.getNoReturn()); ID.AddBoolean(info.getProducesResult()); ID.AddBoolean(info.getNoCallerSavedRegs()); diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h --- a/clang/lib/CodeGen/CGCall.h +++ b/clang/lib/CodeGen/CGCall.h @@ -386,6 +386,35 @@ const TargetOptions &TargetOpts, bool WillInternalize); +enum class FnInfoOpts { + None = 0, + IsInstanceMethod = 1 << 0, + IsChainCall = 1 << 1, + IsDelegateCall = 1 << 2, +}; + +inline FnInfoOpts operator|(FnInfoOpts A, FnInfoOpts B) { + return static_cast( + static_cast>(A) | + static_cast>(B)); +} + +inline FnInfoOpts operator&(FnInfoOpts A, FnInfoOpts B) { + return static_cast( + static_cast>(A) & + static_cast>(B)); +} + +inline FnInfoOpts operator|=(FnInfoOpts A, FnInfoOpts B) { + A = A | B; + return A; +} + +inline FnInfoOpts operator&=(FnInfoOpts A, FnInfoOpts B) { + A = A & B; + return A; +} + } // end namespace CodeGen } // end namespace clang diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -111,8 +111,7 @@ // When translating an unprototyped function type, always use a // variadic type. return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(), - /*instanceMethod=*/false, - /*chainCall=*/false, std::nullopt, + FnInfoOpts::None, std::nullopt, FTNP->getExtInfo(), {}, RequiredArgs(0)); } @@ -188,10 +187,10 @@ appendParameterTypes(CGT, prefix, paramInfos, FTP); CanQualType resultType = FTP->getReturnType().getUnqualifiedType(); - return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod, - /*chainCall=*/false, prefix, - FTP->getExtInfo(), paramInfos, - Required); + FnInfoOpts opts = + instanceMethod ? FnInfoOpts::IsInstanceMethod : FnInfoOpts::None; + return CGT.arrangeLLVMFunctionInfo(resultType, opts, prefix, + FTP->getExtInfo(), paramInfos, Required); } /// Arrange the argument and result information for a value of the @@ -270,7 +269,7 @@ argTypes.push_back(DeriveThisType(RD, MD)); return ::arrangeLLVMFunctionInfo( - *this, true, argTypes, + *this, /*instanceMethod=*/true, argTypes, FTP->getCanonicalTypeUnqualified().getAs()); } @@ -362,9 +361,8 @@ : TheCXXABI.hasMostDerivedReturn(GD) ? CGM.getContext().VoidPtrTy : Context.VoidTy; - return arrangeLLVMFunctionInfo(resultType, /*instanceMethod=*/true, - /*chainCall=*/false, argTypes, extInfo, - paramInfos, required); + return arrangeLLVMFunctionInfo(resultType, FnInfoOpts::IsInstanceMethod, + argTypes, extInfo, paramInfos, required); } static SmallVector @@ -438,9 +436,9 @@ addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs, ArgTypes.size()); } - return arrangeLLVMFunctionInfo(ResultType, /*instanceMethod=*/true, - /*chainCall=*/false, ArgTypes, Info, - ParamInfos, Required); + + return arrangeLLVMFunctionInfo(ResultType, FnInfoOpts::IsInstanceMethod, + ArgTypes, Info, ParamInfos, Required); } /// Arrange the argument and result information for the declaration or @@ -459,10 +457,9 @@ // When declaring a function without a prototype, always use a // non-variadic type. if (CanQual noProto = FTy.getAs()) { - return arrangeLLVMFunctionInfo( - noProto->getReturnType(), /*instanceMethod=*/false, - /*chainCall=*/false, std::nullopt, noProto->getExtInfo(), {}, - RequiredArgs::All); + return arrangeLLVMFunctionInfo(noProto->getReturnType(), FnInfoOpts::None, + std::nullopt, noProto->getExtInfo(), {}, + RequiredArgs::All); } return arrangeFreeFunctionType(FTy.castAs()); @@ -511,9 +508,9 @@ RequiredArgs required = (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All); - return arrangeLLVMFunctionInfo( - GetReturnType(MD->getReturnType()), /*instanceMethod=*/false, - /*chainCall=*/false, argTys, einfo, extParamInfos, required); + return arrangeLLVMFunctionInfo(GetReturnType(MD->getReturnType()), + FnInfoOpts::None, argTys, einfo, extParamInfos, + required); } const CGFunctionInfo & @@ -522,9 +519,8 @@ auto argTypes = getArgTypesForCall(Context, args); FunctionType::ExtInfo einfo; - return arrangeLLVMFunctionInfo( - GetReturnType(returnType), /*instanceMethod=*/false, - /*chainCall=*/false, argTypes, einfo, {}, RequiredArgs::All); + return arrangeLLVMFunctionInfo(GetReturnType(returnType), FnInfoOpts::None, + argTypes, einfo, {}, RequiredArgs::All); } const CGFunctionInfo & @@ -549,8 +545,7 @@ assert(MD->isVirtual() && "only methods have thunks"); CanQual FTP = GetFormalType(MD); CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)}; - return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/false, - /*chainCall=*/false, ArgTys, + return arrangeLLVMFunctionInfo(Context.VoidTy, FnInfoOpts::None, ArgTys, FTP->getExtInfo(), {}, RequiredArgs(1)); } @@ -569,9 +564,8 @@ ArgTys.push_back(Context.IntTy); CallingConv CC = Context.getDefaultCallingConvention( /*IsVariadic=*/false, /*IsCXXMethod=*/true); - return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true, - /*chainCall=*/false, ArgTys, - FunctionType::ExtInfo(CC), {}, + return arrangeLLVMFunctionInfo(Context.VoidTy, FnInfoOpts::IsInstanceMethod, + ArgTys, FunctionType::ExtInfo(CC), {}, RequiredArgs::All); } @@ -615,10 +609,10 @@ SmallVector argTypes; for (const auto &arg : args) argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty)); + FnInfoOpts opts = chainCall ? FnInfoOpts::IsChainCall : FnInfoOpts::None; return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()), - /*instanceMethod=*/false, chainCall, - argTypes, fnType->getExtInfo(), paramInfos, - required); + opts, argTypes, fnType->getExtInfo(), + paramInfos, required); } /// Figure out the rules for calling a function with the given formal @@ -649,8 +643,8 @@ auto argTypes = getArgTypesForDeclaration(Context, params); return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()), - /*instanceMethod*/ false, /*chainCall*/ false, - argTypes, proto->getExtInfo(), paramInfos, + FnInfoOpts::None, argTypes, + proto->getExtInfo(), paramInfos, RequiredArgs::forPrototypePlus(proto, 1)); } @@ -661,10 +655,9 @@ SmallVector argTypes; for (const auto &Arg : args) argTypes.push_back(Context.getCanonicalParamType(Arg.Ty)); - return arrangeLLVMFunctionInfo( - GetReturnType(resultType), /*instanceMethod=*/false, - /*chainCall=*/false, argTypes, FunctionType::ExtInfo(), - /*paramInfos=*/ {}, RequiredArgs::All); + return arrangeLLVMFunctionInfo(GetReturnType(resultType), FnInfoOpts::None, + argTypes, FunctionType::ExtInfo(), + /*paramInfos=*/{}, RequiredArgs::All); } const CGFunctionInfo & @@ -672,17 +665,17 @@ const FunctionArgList &args) { auto argTypes = getArgTypesForDeclaration(Context, args); - return arrangeLLVMFunctionInfo( - GetReturnType(resultType), /*instanceMethod=*/false, /*chainCall=*/false, - argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All); + return arrangeLLVMFunctionInfo(GetReturnType(resultType), FnInfoOpts::None, + argTypes, FunctionType::ExtInfo(), {}, + RequiredArgs::All); } const CGFunctionInfo & CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType, ArrayRef argTypes) { - return arrangeLLVMFunctionInfo( - resultType, /*instanceMethod=*/false, /*chainCall=*/false, - argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All); + return arrangeLLVMFunctionInfo(resultType, FnInfoOpts::None, argTypes, + FunctionType::ExtInfo(), {}, + RequiredArgs::All); } /// Arrange a call to a C++ method, passing the given arguments. @@ -705,15 +698,15 @@ auto argTypes = getArgTypesForCall(Context, args); FunctionType::ExtInfo info = proto->getExtInfo(); - return arrangeLLVMFunctionInfo( - GetReturnType(proto->getReturnType()), /*instanceMethod=*/true, - /*chainCall=*/false, argTypes, info, paramInfos, required); + return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()), + FnInfoOpts::IsInstanceMethod, argTypes, info, + paramInfos, required); } const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() { - return arrangeLLVMFunctionInfo( - getContext().VoidTy, /*instanceMethod=*/false, /*chainCall=*/false, - std::nullopt, FunctionType::ExtInfo(), {}, RequiredArgs::All); + return arrangeLLVMFunctionInfo(getContext().VoidTy, FnInfoOpts::None, + std::nullopt, FunctionType::ExtInfo(), {}, + RequiredArgs::All); } const CGFunctionInfo & @@ -733,12 +726,15 @@ auto argTypes = getArgTypesForCall(Context, args); assert(signature.getRequiredArgs().allowsOptionalArgs()); - return arrangeLLVMFunctionInfo(signature.getReturnType(), - signature.isInstanceMethod(), - signature.isChainCall(), - argTypes, - signature.getExtInfo(), - paramInfos, + FnInfoOpts opts = FnInfoOpts::None; + if (signature.isInstanceMethod()) + opts |= FnInfoOpts::IsInstanceMethod; + if (signature.isChainCall()) + opts |= FnInfoOpts::IsChainCall; + if (signature.isDelegateCall()) + opts |= FnInfoOpts::IsDelegateCall; + return arrangeLLVMFunctionInfo(signature.getReturnType(), opts, argTypes, + signature.getExtInfo(), paramInfos, signature.getRequiredArgs()); } @@ -751,21 +747,24 @@ /// Arrange the argument and result information for an abstract value /// of a given function type. This is the method which all of the /// above functions ultimately defer to. -const CGFunctionInfo & -CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType, - bool instanceMethod, - bool chainCall, - ArrayRef argTypes, - FunctionType::ExtInfo info, - ArrayRef paramInfos, - RequiredArgs required) { +const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo( + CanQualType resultType, FnInfoOpts opts, ArrayRef argTypes, + FunctionType::ExtInfo info, + ArrayRef paramInfos, + RequiredArgs required) { assert(llvm::all_of(argTypes, [](CanQualType T) { return T.isCanonicalAsParam(); })); // Lookup or create unique function info. llvm::FoldingSetNodeID ID; - CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos, - required, resultType, argTypes); + bool isInstanceMethod = + (opts & FnInfoOpts::IsInstanceMethod) == FnInfoOpts::IsInstanceMethod; + bool isChainCall = + (opts & FnInfoOpts::IsChainCall) == FnInfoOpts::IsChainCall; + bool isDelegateCall = + (opts & FnInfoOpts::IsDelegateCall) == FnInfoOpts::IsDelegateCall; + CGFunctionInfo::Profile(ID, isInstanceMethod, isChainCall, isDelegateCall, + info, paramInfos, required, resultType, argTypes); void *insertPos = nullptr; CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos); @@ -775,8 +774,8 @@ unsigned CC = ClangCallConvToLLVMCallConv(info.getCC()); // Construct the function info. We co-allocate the ArgInfos. - FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info, - paramInfos, resultType, argTypes, required); + FI = CGFunctionInfo::create(CC, isInstanceMethod, isChainCall, isDelegateCall, + info, paramInfos, resultType, argTypes, required); FunctionInfos.InsertNode(FI, insertPos); bool inserted = FunctionsBeingProcessed.insert(FI).second; @@ -811,9 +810,8 @@ return *FI; } -CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, - bool instanceMethod, - bool chainCall, +CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, bool instanceMethod, + bool chainCall, bool delegateCall, const FunctionType::ExtInfo &info, ArrayRef paramInfos, CanQualType resultType, @@ -833,6 +831,7 @@ FI->ASTCallingConvention = info.getCC(); FI->InstanceMethod = instanceMethod; FI->ChainCall = chainCall; + FI->DelegateCall = delegateCall; FI->CmseNSCall = info.getCmseNSCall(); FI->NoReturn = info.getNoReturn(); FI->ReturnsRetained = info.getProducesResult(); @@ -3985,10 +3984,6 @@ QualType type = param->getType(); - if (isInAllocaArgument(CGM.getCXXABI(), type)) { - CGM.ErrorUnsupported(param, "forwarded non-trivially copyable parameter"); - } - // GetAddrOfLocalVar returns a pointer-to-pointer for references, // but the argument needs to be the original pointer. if (type->isReferenceType()) { diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp --- a/clang/lib/CodeGen/CGClass.cpp +++ b/clang/lib/CodeGen/CGClass.cpp @@ -2929,14 +2929,16 @@ } void CodeGenFunction::EmitForwardingCallToLambda( - const CXXMethodDecl *callOperator, - CallArgList &callArgs) { + const CXXMethodDecl *callOperator, CallArgList &callArgs, + const CGFunctionInfo *calleeFnInfo, llvm::Constant *calleePtr) { // Get the address of the call operator. - const CGFunctionInfo &calleeFnInfo = - CGM.getTypes().arrangeCXXMethodDeclaration(callOperator); - llvm::Constant *calleePtr = - CGM.GetAddrOfFunction(GlobalDecl(callOperator), - CGM.getTypes().GetFunctionType(calleeFnInfo)); + if (!calleeFnInfo) + calleeFnInfo = &CGM.getTypes().arrangeCXXMethodDeclaration(callOperator); + + if (!calleePtr) + calleePtr = + CGM.GetAddrOfFunction(GlobalDecl(callOperator), + CGM.getTypes().GetFunctionType(*calleeFnInfo)); // Prepare the return slot. const FunctionProtoType *FPT = @@ -2944,8 +2946,8 @@ QualType resultType = FPT->getReturnType(); ReturnValueSlot returnSlot; if (!resultType->isVoidType() && - calleeFnInfo.getReturnInfo().getKind() == ABIArgInfo::Indirect && - !hasScalarEvaluationKind(calleeFnInfo.getReturnType())) + calleeFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect && + !hasScalarEvaluationKind(calleeFnInfo->getReturnType())) returnSlot = ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(), /*IsUnused=*/false, /*IsExternallyDestructed=*/true); @@ -2956,7 +2958,7 @@ // Now emit our call. auto callee = CGCallee::forDirect(calleePtr, GlobalDecl(callOperator)); - RValue RV = EmitCall(calleeFnInfo, callee, returnSlot, callArgs); + RValue RV = EmitCall(*calleeFnInfo, callee, returnSlot, callArgs); // If necessary, copy the returned value into the slot. if (!resultType->isVoidType() && returnSlot.isNull()) { @@ -2998,7 +3000,15 @@ EmitForwardingCallToLambda(CallOp, CallArgs); } -void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD) { +void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) { + if (MD->isVariadic()) { + // FIXME: Making this work correctly is nasty because it requires either + // cloning the body of the call operator or making the call operator + // forward. + CGM.ErrorUnsupported(MD, "lambda conversion to variadic function"); + return; + } + const CXXRecordDecl *Lambda = MD->getParent(); // Start building arguments for forwarding call @@ -3009,10 +3019,16 @@ Address ThisPtr = CreateMemTemp(LambdaType, "unused.capture"); CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType); - // Add the rest of the parameters. + EmitLambdaDelegatingInvokeBody(MD, CallArgs); +} + +void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD, + CallArgList &CallArgs) { + // Add the rest of the forwarded parameters. for (auto *Param : MD->parameters()) EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc()); + const CXXRecordDecl *Lambda = MD->getParent(); const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator(); // For a generic lambda, find the corresponding call operator specialization // to which the call to the static-invoker shall be forwarded. @@ -3026,10 +3042,21 @@ assert(CorrespondingCallOpSpecialization); CallOp = cast(CorrespondingCallOpSpecialization); } + + // Special lambda forwarding when there are inalloca parameters. + if (hasInAllocaArg(MD)) { + const CGFunctionInfo *ImplFnInfo = nullptr; + llvm::Function *ImplFn = nullptr; + EmitLambdaInAllocaImplFn(CallOp, &ImplFnInfo, &ImplFn); + + EmitForwardingCallToLambda(CallOp, CallArgs, ImplFnInfo, ImplFn); + return; + } + EmitForwardingCallToLambda(CallOp, CallArgs); } -void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) { +void CodeGenFunction::EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD) { if (MD->isVariadic()) { // FIXME: Making this work correctly is nasty because it requires either // cloning the body of the call operator or making the call operator forward. @@ -3037,5 +3064,49 @@ return; } - EmitLambdaDelegatingInvokeBody(MD); + // Forward %this argument. + CallArgList CallArgs; + QualType LambdaType = getContext().getRecordType(MD->getParent()); + QualType ThisType = getContext().getPointerType(LambdaType); + llvm::Value *ThisArg = CurFn->getArg(0); + CallArgs.add(RValue::get(ThisArg), ThisType); + + EmitLambdaDelegatingInvokeBody(MD, CallArgs); +} + +void CodeGenFunction::EmitLambdaInAllocaImplFn( + const CXXMethodDecl *CallOp, const CGFunctionInfo **ImplFnInfo, + llvm::Function **ImplFn) { + const CGFunctionInfo &FnInfo = + CGM.getTypes().arrangeCXXMethodDeclaration(CallOp); + llvm::Function *CallOpFn = + cast(CGM.GetAddrOfFunction(GlobalDecl(CallOp))); + + // Emit function containing the original call op body. __invoke will delegate + // to this function. + SmallVector ArgTypes; + for (auto I = FnInfo.arg_begin(); I != FnInfo.arg_end(); ++I) + ArgTypes.push_back(I->type); + *ImplFnInfo = &CGM.getTypes().arrangeLLVMFunctionInfo( + FnInfo.getReturnType(), FnInfoOpts::IsDelegateCall, ArgTypes, + FnInfo.getExtInfo(), {}, FnInfo.getRequiredArgs()); + + // Create mangled name as if this was a method named __impl. + StringRef CallOpName = CallOpFn->getName(); + std::string ImplName = + ("?__impl@" + CallOpName.drop_front(CallOpName.find_first_of("<"))).str(); + + llvm::Function *Fn = CallOpFn->getParent()->getFunction(ImplName); + if (!Fn) { + Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(**ImplFnInfo), + llvm::GlobalValue::InternalLinkage, ImplName, + CGM.getModule()); + CGM.SetInternalFunctionAttributes(CallOp, Fn, **ImplFnInfo); + + const GlobalDecl &GD = GlobalDecl(CallOp); + const auto *D = cast(GD.getDecl()); + CodeGenFunction(CGM).GenerateCode(GD, Fn, **ImplFnInfo); + CGM.SetLLVMFunctionAttributesForDefinition(D, Fn); + } + *ImplFn = Fn; } diff --git a/clang/lib/CodeGen/CGDeclCXX.cpp b/clang/lib/CodeGen/CGDeclCXX.cpp --- a/clang/lib/CodeGen/CGDeclCXX.cpp +++ b/clang/lib/CodeGen/CGDeclCXX.cpp @@ -279,8 +279,8 @@ } const CGFunctionInfo &FI = CGM.getTypes().arrangeLLVMFunctionInfo( - getContext().IntTy, /*instanceMethod=*/false, /*chainCall=*/false, - {getContext().IntTy}, FunctionType::ExtInfo(), {}, RequiredArgs::All); + getContext().IntTy, FnInfoOpts::None, {getContext().IntTy}, + FunctionType::ExtInfo(), {}, RequiredArgs::All); // Get the stub function type, int(*)(int,...). llvm::FunctionType *StubTy = diff --git a/clang/lib/CodeGen/CodeGenABITypes.cpp b/clang/lib/CodeGen/CodeGenABITypes.cpp --- a/clang/lib/CodeGen/CodeGenABITypes.cpp +++ b/clang/lib/CodeGen/CodeGenABITypes.cpp @@ -65,9 +65,8 @@ ArrayRef argTypes, FunctionType::ExtInfo info, RequiredArgs args) { - return CGM.getTypes().arrangeLLVMFunctionInfo( - returnType, /*instanceMethod=*/false, /*chainCall=*/false, argTypes, - info, {}, args); + return CGM.getTypes().arrangeLLVMFunctionInfo(returnType, FnInfoOpts::None, + argTypes, info, {}, args); } ImplicitCXXConstructorArgs diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -1963,6 +1963,9 @@ /// Check if the return value of this function requires sanitization. bool requiresReturnValueCheck() const; + bool isInAllocaArgument(CGCXXABI &ABI, QualType Ty); + bool hasInAllocaArg(const CXXMethodDecl *MD); + llvm::BasicBlock *TerminateLandingPad = nullptr; llvm::BasicBlock *TerminateHandler = nullptr; llvm::SmallVector TrapBBs; @@ -2225,10 +2228,17 @@ void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S); void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator, - CallArgList &CallArgs); + CallArgList &CallArgs, + const CGFunctionInfo *CallOpFnInfo = nullptr, + llvm::Constant *CallOpFn = nullptr); void EmitLambdaBlockInvokeBody(); - void EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD); void EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD); + void EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD, + CallArgList &CallArgs); + void EmitLambdaInAllocaImplFn(const CXXMethodDecl *CallOp, + const CGFunctionInfo **ImplFnInfo, + llvm::Function **ImplFn); + void EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD); void EmitLambdaVLACapture(const VariableArrayType *VAT, LValue LV) { EmitStoreThroughLValue(RValue::get(VLASizeMap[VAT->getSizeExpr()]), LV); } diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -683,6 +683,19 @@ return true; } +bool CodeGenFunction::isInAllocaArgument(CGCXXABI &ABI, QualType Ty) { + const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl(); + return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory; +} + +bool CodeGenFunction::hasInAllocaArg(const CXXMethodDecl *MD) { + return getTarget().getTriple().getArch() == llvm::Triple::x86 && + getTarget().getCXXABI().isMicrosoft() && + llvm::any_of(MD->parameters(), [&](ParmVarDecl *P) { + return isInAllocaArgument(CGM.getCXXABI(), P->getType()); + }); +} + /// Return the UBSan prologue signature for \p FD if one is available. static llvm::Constant *getPrologueSignature(CodeGenModule &CGM, const FunctionDecl *FD) { @@ -1447,6 +1460,17 @@ // The lambda static invoker function is special, because it forwards or // clones the body of the function call operator (but is actually static). EmitLambdaStaticInvokeBody(cast(FD)); + + } else if (isa(FD) && + isLambdaCallOperator(cast(FD)) && + cast(FD)->getParent()->getLambdaStaticInvoker() && + hasInAllocaArg(cast(FD) + ->getParent() + ->getLambdaStaticInvoker()) && + !FnInfo.isDelegateCall()) { + // If emitting a lambda with static invoker on X86 Windows, change + // the call operator body. + EmitLambdaInAllocaCallOpBody(cast(FD)); } else if (FD->isDefaulted() && isa(FD) && (cast(FD)->isCopyAssignmentOperator() || cast(FD)->isMoveAssignmentOperator())) { diff --git a/clang/lib/CodeGen/CodeGenTypes.h b/clang/lib/CodeGen/CodeGenTypes.h --- a/clang/lib/CodeGen/CodeGenTypes.h +++ b/clang/lib/CodeGen/CodeGenTypes.h @@ -252,13 +252,11 @@ /// this. /// /// \param argTypes - must all actually be canonical as params - const CGFunctionInfo &arrangeLLVMFunctionInfo(CanQualType returnType, - bool instanceMethod, - bool chainCall, - ArrayRef argTypes, - FunctionType::ExtInfo info, - ArrayRef paramInfos, - RequiredArgs args); + const CGFunctionInfo &arrangeLLVMFunctionInfo( + CanQualType returnType, FnInfoOpts opts, ArrayRef argTypes, + FunctionType::ExtInfo info, + ArrayRef paramInfos, + RequiredArgs args); /// Compute a new LLVM record layout object for the given record. std::unique_ptr ComputeRecordLayout(const RecordDecl *D, diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp --- a/clang/lib/CodeGen/Targets/X86.cpp +++ b/clang/lib/CodeGen/Targets/X86.cpp @@ -140,7 +140,8 @@ Class classify(QualType Ty) const; ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const; - ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State) const; + ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State, + bool isDelegateCall) const; /// Updates the number of available free registers, returns /// true if any registers were allocated. @@ -738,8 +739,8 @@ } } -ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, - CCState &State) const { +ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, CCState &State, + bool isDelegateCall) const { // FIXME: Set alignment on indirect arguments. bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall; bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall; @@ -752,7 +753,7 @@ const RecordType *RT = Ty->getAs(); if (RT) { CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()); - if (RAA == CGCXXABI::RAA_Indirect) { + if (RAA == CGCXXABI::RAA_Indirect || isDelegateCall) { return getIndirectResult(Ty, false, State); } else if (RAA == CGCXXABI::RAA_DirectInMemory) { // The field index doesn't matter, we'll fix it up later. @@ -941,7 +942,8 @@ if (State.IsPreassigned.test(I)) continue; - Args[I].info = classifyArgumentType(Args[I].type, State); + Args[I].info = + classifyArgumentType(Args[I].type, State, FI.isDelegateCall()); UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca); } diff --git a/clang/test/CodeGenCXX/inalloca-lambda.cpp b/clang/test/CodeGenCXX/inalloca-lambda.cpp --- a/clang/test/CodeGenCXX/inalloca-lambda.cpp +++ b/clang/test/CodeGenCXX/inalloca-lambda.cpp @@ -1,11 +1,50 @@ -// RUN: not %clang_cc1 -triple i686-windows-msvc -emit-llvm -o /dev/null %s 2>&1 | FileCheck %s +// RUN: %clang_cc1 -triple i686-windows-msvc -emit-llvm -o - %s 2>&1 | FileCheck %s -// PR28299 -// CHECK: error: cannot compile this forwarded non-trivially copyable parameter yet - -class A { +struct A { + A(); A(const A &); + int x; }; -typedef void (*fptr_t)(A); -fptr_t fn1() { return [](A) {}; } +void decayToFp(int (*f)(A)); +void test() { + auto ld = [](A a) { + static int calls = 0; + ++calls; + return a.x + calls; + }; + decayToFp(ld); + ld(A{}); +} + +// CHECK: define internal x86_thiscallcc noundef i32 +// CHECK-SAME: @"??R@?0??test@@YAXXZ@QBE?A?@@UA@@@Z" +// CHECK-SAME: (ptr noundef %this, ptr inalloca(<{ %struct.A }>) %[[ARG:.*]]) +// CHECK: %[[V:.*]] = getelementptr inbounds <{ %struct.A }>, ptr %[[ARG]], i32 0, i32 0 +// CHECK: %call = call x86_thiscallcc noundef i32 +// CHECK-SAME: @"?__impl@@?0??test@@YAXXZ@QBE?A?@@UA@@@Z" +// CHECK-SAME: (ptr noundef %this, ptr noundef %[[V]]) + +// CHECK: define internal noundef i32 +// CHECK-SAME: @"?__invoke@@?0??test@@YAXXZ@CA?A?@@UA@@@Z" +// CHECK-SAME: (ptr inalloca(<{ %struct.A }>) %[[ARG:.*]]) +// CHECK: %unused.capture = alloca %class.anon, align 1 +// CHECK: %[[VAR:.*]] = getelementptr inbounds <{ %struct.A }>, ptr %[[ARG]], i32 0, i32 0 +// CHECK: %call = call x86_thiscallcc noundef i32 +// CHECK-SAME: @"?__impl@@?0??test@@YAXXZ@QBE?A?@@UA@@@Z" +// CHECK-SAME: (ptr noundef %unused.capture, ptr noundef %[[VAR]]) +// CHECK: ret i32 %call +// CHECK: define internal x86_thiscallcc noundef i32 +// CHECK-SAME: @"?__impl@@?0??test@@YAXXZ@QBE?A?@@UA@@@Z" +// CHECK-SAME: (ptr noundef %this, ptr noundef %[[ARG:.*]]) +// CHECK: %this.addr = alloca ptr, align 4 +// CHECK: store ptr %this, ptr %this.addr, align 4 +// CHECK: %this1 = load ptr, ptr %this.addr, align 4 +// CHECK: %{{.*}} = load i32, ptr @"?calls@?1???R +// CHECK: %inc = add nsw i32 %{{.*}}, 1 +// CHECK: store i32 %inc, ptr @"?calls@?1???R +// CHECK: %{{.*}} = getelementptr inbounds %struct.A, ptr %{{.*}}, i32 0, i32 0 +// CHECK: %{{.*}} = load i32, ptr %{{.*}}, align 4 +// CHECK: %{{.*}} = load i32, ptr @"?calls@?1???R +// CHECK: %add = add nsw i32 %{{.*}}, %{{.*}} +// CHECK: ret i32 %add