diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -3805,10 +3805,6 @@ QualType type = param->getType(); - if (isInAllocaArgument(CGM.getCXXABI(), type)) { - CGM.ErrorUnsupported(param, "forwarded non-trivially copyable parameter"); - } - // GetAddrOfLocalVar returns a pointer-to-pointer for references, // but the argument needs to be the original pointer. if (type->isReferenceType()) { diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp --- a/clang/lib/CodeGen/CGClass.cpp +++ b/clang/lib/CodeGen/CGClass.cpp @@ -3003,21 +3003,41 @@ EmitForwardingCallToLambda(CallOp, CallArgs); } +static bool isInAllocaArg(CGCXXABI &ABI, QualType T) { + const CXXRecordDecl *RD = T->getAsCXXRecordDecl(); + return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory; +} + +static bool hasInAllocaArg(const TargetInfo &TI, CGCXXABI &ABI, const CXXMethodDecl *MD) { + return TI.getCXXABI().isMicrosoft() && + llvm::any_of(MD->parameters(), [&](ParmVarDecl *P) { + return isInAllocaArg(ABI, P->getType()); + }); +} + void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD) { const CXXRecordDecl *Lambda = MD->getParent(); // Start building arguments for forwarding call CallArgList CallArgs; + SmallVector ArgList; QualType LambdaType = getContext().getRecordType(Lambda); QualType ThisType = getContext().getPointerType(LambdaType); Address ThisPtr = CreateMemTemp(LambdaType, "unused.capture"); CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType); + ArgList.push_back(ThisPtr.getPointer()); // Add the rest of the parameters. - for (auto *Param : MD->parameters()) + for (auto *Param : MD->parameters()) { EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc()); + if (Param->getType()->isReferenceType()) + ArgList.push_back(Builder.CreateLoad(GetAddrOfLocalVar(Param))); + else + ArgList.push_back(GetAddrOfLocalVar(Param).getPointer()); + } + const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator(); // For a generic lambda, find the corresponding call operator specialization // to which the call to the static-invoker shall be forwarded. @@ -3031,9 +3051,81 @@ assert(CorrespondingCallOpSpecialization); CallOp = cast(CorrespondingCallOpSpecialization); } + + // Special lambda forwarding when there are inalloca parameters. + if (hasInAllocaArg(getTarget(), CGM.getCXXABI(), MD)) { + EmitInAllocaForwardingCallToLambda(MD, CallOp, ArgList, CallArgs); + return; + } + EmitForwardingCallToLambda(CallOp, CallArgs); } +void CodeGenFunction::EmitInAllocaForwardingCallToLambda( + const CXXMethodDecl *MD, const CXXMethodDecl *CallOp, + ArrayRef ArgList, + CallArgList &CallArgs) { + const CGFunctionInfo &FnInfo = + CGM.getTypes().arrangeCXXMethodDeclaration(CallOp); + + // Create new call op function with different calling convention. + CodeGenFunction CGF(CGM); + CGF.CurGD = GlobalDecl(CallOp); + llvm::Function *CallOpFn = + cast(CGM.GetAddrOfFunction(GlobalDecl(CallOp))); + + std::string ImplName = (CallOpFn->getName() + ".impl").str(); + llvm::Function *ImplFn = CallOpFn->getParent()->getFunction(ImplName); + if (!ImplFn) { + // Generate the function. + ImplFn = llvm::Function::Create(CGM.getTypes().GetFunctionType(FnInfo), + llvm::GlobalValue::InternalLinkage, + ImplName, CGM.getModule()); + ImplFn->setCallingConv(llvm::CallingConv::C); + CGF.GenerateCode(CGF.CurGD, ImplFn, FnInfo); + CGM.SetLLVMFunctionAttributesForDefinition(CallOp, ImplFn); + } + + // Deactivate any cleanups that we're supposed to do immediately before + // the call. + if (!CallArgs.getCleanupsToDeactivate().empty()) { + ArrayRef Cleanups = + CallArgs.getCleanupsToDeactivate(); + for (const auto &I : llvm::reverse(Cleanups)) { + DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP); + I.IsActiveIP->eraseFromParent(); + } + } + + // Get the args to pass to the call op. This only works because we're assuming + // there are two arguments; this, and the inalloca one. + SmallVector NewArgList; + NewArgList.push_back(ArgList[0]); // Add the %this arg. + CGFunctionInfo::const_arg_iterator info_it = FnInfo.arg_begin(); + for (auto I = ArgList.begin(); I != ArgList.end(); ++I, ++info_it) { + const ABIArgInfo &ArgInfo = info_it->info; + if (ArgInfo.getKind() == ABIArgInfo::InAlloca) { + NewArgList.push_back(*I); // Add the inalloca arg. + break; + } + } + + // Create the call. + llvm::CallInst *Call = Builder.CreateCall(ImplFn, ArgList); + Call->setCallingConv(ImplFn->getCallingConv()); + + QualType RetTy = FnInfo.getReturnType(); + ReturnValueSlot ReturnSlot; + + // If necessary, copy the returned value into the slot. + if (!RetTy->isVoidType() && ReturnSlot.isNull()) + EmitReturnOfRValue(RValue::get(Call), RetTy); + else + EmitBranchThroughCleanup(ReturnBlock); + + return; +} + void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) { if (MD->isVariadic()) { // FIXME: Making this work correctly is nasty because it requires either @@ -3041,6 +3133,5 @@ CGM.ErrorUnsupported(MD, "lambda conversion to variadic function"); return; } - EmitLambdaDelegatingInvokeBody(MD); } diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -2223,6 +2223,10 @@ void EmitLambdaBlockInvokeBody(); void EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD); void EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD); + void EmitInAllocaForwardingCallToLambda(const CXXMethodDecl *MD, + const CXXMethodDecl *CallOp, + ArrayRef ArgList, + CallArgList &CallArgs); void EmitLambdaVLACapture(const VariableArrayType *VAT, LValue LV) { EmitStoreThroughLValue(RValue::get(VLASizeMap[VAT->getSizeExpr()]), LV); }