Index: include/clang/AST/ASTContext.h =================================================================== --- include/clang/AST/ASTContext.h +++ include/clang/AST/ASTContext.h @@ -2065,6 +2065,9 @@ /// Get default simd alignment of the specified complete type in bits. unsigned getOpenMPDefaultSimdAlign(QualType T) const; + /// Get maximum potential size of a coroutine frame for the given lambda type. + unsigned getCoroFrameMaxSize(QualType T) const; + /// Return the size of the specified (complete) type \p T, in bits. uint64_t getTypeSize(QualType T) const { return getTypeInfo(T).Width; } uint64_t getTypeSize(const Type *T) const { return getTypeInfo(T).Width; } Index: include/clang/AST/DeclCXX.h =================================================================== --- include/clang/AST/DeclCXX.h +++ include/clang/AST/DeclCXX.h @@ -2180,7 +2180,7 @@ /// that for the call operator of a lambda closure type, this returns the /// desugared 'this' type (a pointer to the closure type), not the captured /// 'this' type. - QualType getThisType(ASTContext &C) const; + QualType getThisType(const ASTContext &C) const; static QualType getThisType(const FunctionProtoType *FPT, const CXXRecordDecl *Decl); Index: include/clang/Basic/DiagnosticSemaKinds.td =================================================================== --- include/clang/Basic/DiagnosticSemaKinds.td +++ include/clang/Basic/DiagnosticSemaKinds.td @@ -5438,7 +5438,7 @@ // Expressions. def select_unary_expr_or_type_trait_kind : TextSubstitution< "%select{sizeof|alignof|vec_step|__builtin_omp_required_simd_align|" - "__alignof}0">; + "__alignof|__builtin_coro_frame_max_size}0">; def ext_sizeof_alignof_function_type : Extension< "invalid application of '%sub{select_unary_expr_or_type_trait_kind}0' " "to a function type">, InGroup; @@ -5456,12 +5456,20 @@ "to a function type">; def err_openmp_default_simd_align_expr : Error< "invalid application of '__builtin_omp_required_simd_align' to an expression, only type is allowed">; +def err_coro_frame_max_size_expr : Error< + "invalid application of '__builtin_coro_frame_max_size' to an expression, only type is allowed">; def err_sizeof_alignof_typeof_bitfield : Error< "invalid application of '%select{sizeof|alignof|typeof}0' to bit-field">; def err_alignof_member_of_incomplete_type : Error< "invalid application of 'alignof' to a field of a class still being defined">; def err_vecstep_non_scalar_vector_type : Error< "'vec_step' requires built-in scalar or vector type, %0 invalid">; +def err_coro_frame_max_size_non_lambda_type : Error< + "invalid application of '__builtin_coro_frame_max_size' to a non-lambda type %0">; +def err_coro_frame_max_size_not_coroutine_lambda : Error< + "invalid application of '__builtin_coro_frame_max_size' to a non-coroutine lambda type %0">; +def err_coro_frame_max_size_lambda_missing_body : Error< + "invalid application of '__builtin_coro_frame_max_size' to a lambda type %0 without a body">; def err_offsetof_incomplete_type : Error< "offsetof of incomplete type %0">; def err_offsetof_record_type : Error< Index: include/clang/Basic/TokenKinds.def =================================================================== --- include/clang/Basic/TokenKinds.def +++ include/clang/Basic/TokenKinds.def @@ -372,9 +372,10 @@ CONCEPTS_KEYWORD(requires) // C++ coroutines TS keywords -KEYWORD(co_await , KEYCOROUTINES) -KEYWORD(co_return , KEYCOROUTINES) -KEYWORD(co_yield , KEYCOROUTINES) +KEYWORD(co_await , KEYCOROUTINES) +KEYWORD(co_return , KEYCOROUTINES) +KEYWORD(co_yield , KEYCOROUTINES) +KEYWORD(__builtin_coro_frame_max_size, KEYCOROUTINES) // C++ modules TS keywords MODULES_KEYWORD(module) Index: include/clang/Basic/TypeTraits.h =================================================================== --- include/clang/Basic/TypeTraits.h +++ include/clang/Basic/TypeTraits.h @@ -105,6 +105,7 @@ /// __alignof returns the preferred alignment of a type, the alignment /// clang will attempt to give an object of the type if allowed by ABI. UETT_PreferredAlignOf, + UETT_CoroFrameMaxSize, }; } Index: lib/AST/ASTContext.cpp =================================================================== --- lib/AST/ASTContext.cpp +++ lib/AST/ASTContext.cpp @@ -2152,6 +2152,140 @@ return SimdAlign; } +namespace { +class FrameSizeBuilder { + const ASTContext &Context; + uint64_t FrameSize = 0; + +public: + FrameSizeBuilder(const ASTContext &Context) : Context(Context) {} + + uint64_t getFrameSize() const { return FrameSize; } + + void addType(const QualType Ty) { + if ((FrameSize & (Context.getTypeAlign(Ty) - 1)) != 0) + FrameSize = llvm::alignTo(FrameSize, + Context.getTypeAlignInChars(Ty).getQuantity()); + FrameSize += Context.getTypeSizeInChars(Ty).getQuantity(); + } + + void addType(TargetInfo::IntType Ty) { + const TargetInfo &TI = Context.getTargetInfo(); + if ((FrameSize & (TI.getTypeAlign(Ty) - 1)) != 0) + FrameSize = llvm::alignTo( + FrameSize, + Context.toCharUnitsFromBits(TI.getTypeAlign(Ty)).getQuantity()); + FrameSize += TI.getTypeWidth(Ty); + } +}; + +class PotentialSpillsVisitor + : public RecursiveASTVisitor { + FrameSizeBuilder &Builder; + const QualType HandleTy; + +public: + PotentialSpillsVisitor(FrameSizeBuilder &Builder, QualType HandleTy) + : Builder(Builder), HandleTy(HandleTy) {} + + bool shouldVisitImplicitCode() const { return true; } + + // Add up the size of any variables explicitly declared within the + // coroutine, as well as the implicit __promise and __coro_gro variables. + bool VisitVarDecl(VarDecl *VD) { + Builder.addType(VD->getType()); + return true; + } + + // Add up the size of any temporaries materialized within the coroutine, as + // well as the implicit temporaries materialized when: + // 1. __promise.initial_suspend and .final_suspend are called to construct + // awaitables + // 2. Coroutine handles are materialized via .from_address and + // passed in as arguments to __builtin_coro_frame. Coroutine handles also + // get bitcast to void* in LLVM IR and the bitcast spills, so the bitcast + // also needs to be accounted for in the coroutine frame size + // 3. __promise.get_return_object is called to construct return objects + bool VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E) { + QualType ETy = E->GetTemporaryExpr()->getType(); + Builder.addType(ETy); + + // LLVM spills the bitcast for this type, causing an additional increase + // in coroutine frame size. + if (ETy == HandleTy) + Builder.addType(ETy); + + return true; + } +}; +}; + +unsigned ASTContext::getCoroFrameMaxSize(QualType T) const { + auto *TTy = cast(T.getTypePtr()); + auto *RTy = cast(TTy->getReplacementType().getTypePtr()); + auto *RD = cast(RTy->getDecl()); + assert(RD->isLambda()); + + CXXMethodDecl *MD = RD->getLambdaCallOperator(); + assert(MD->doesThisDeclarationHaveABody()); + + auto *CoroBody = cast(MD->getBody()); + + // Build the maximum potential coroutine frame size, adding alignment padding + // as necessary. + FrameSizeBuilder Builder(*this); + + // Add space for the two llvm.coro.subfn pointers LLVM adds to the coroutine + // frame. + Builder.addType(VoidPtrTy); + Builder.addType(VoidPtrTy); + + // The coroutine resume index added by LLVM to the coroutine frame will use + // the smallest viable integer type, which in some cases might be i1 (1 byte) + // vs. i64 (8 bytes). So the maximum size here might be 7 bytes larger than + // actual. + Builder.addType(getTargetInfo().getInt64Type()); + + // Add space for the promise that LLVM adds to the coroutine frame. + auto *PromiseDecl = CoroBody->getPromiseDeclStmt(); + QualType PromiseType; + if (auto *PromiseDeclStmt = dyn_cast(PromiseDecl)) + if (auto *PromiseVarDecl = + dyn_cast(PromiseDeclStmt->getSingleDecl())) + PromiseType = PromiseVarDecl->getType(); + Builder.addType(PromiseType); + + // Next, we add space for any and all variables or temporaries that may spill + // across suspend point boundaries, and thus may be moved onto the coroutine + // frame. In reality LLVM may elide many of these moves, but this function + // calculates the upper bound for the frame size, so we must be pessimistic + // and assume all variables end up on the frame. + + // First, the implicit 'this' variable on the lambda. + Builder.addType(MD->getThisType(*this)); + + // Next, each of the parameters to the lambda. + for (auto *PD : MD->parameters()) + Builder.addType(PD->getType()); + + // Finally, any temporaries that may be materialized -- especially coroutine + // handles, for which LLVM introduces a bitcast and so must be counted twice. + QualType HandleTy; + if (auto *InitSuspend = + dyn_cast(CoroBody->getInitSuspendStmt())) + if (auto *Coawait = dyn_cast(InitSuspend->getSubExpr())) + if (auto *AwaitSuspend = + dyn_cast(Coawait->getSuspendExpr())) + if (auto *HandleCtor = + dyn_cast(AwaitSuspend->getArg(0))) + if (auto *Cast = dyn_cast(HandleCtor->getArg(0))) + HandleTy = Cast->getSubExpr()->getType(); + PotentialSpillsVisitor V(Builder, HandleTy); + V.TraverseStmt(CoroBody); + + return Builder.getFrameSize(); +} + /// toCharUnitsFromBits - Convert a size in bits to a size in characters. CharUnits ASTContext::toCharUnitsFromBits(int64_t BitSize) const { return CharUnits::fromQuantity(BitSize / getCharWidth()); Index: lib/AST/ASTDumper.cpp =================================================================== --- lib/AST/ASTDumper.cpp +++ lib/AST/ASTDumper.cpp @@ -1986,6 +1986,9 @@ case UETT_OpenMPRequiredSimdAlign: OS << " __builtin_omp_required_simd_align"; break; + case UETT_CoroFrameMaxSize: + OS << " __builtin_coro_frame_max_size"; + break; case UETT_PreferredAlignOf: OS << " __alignof"; break; Index: lib/AST/DeclCXX.cpp =================================================================== --- lib/AST/DeclCXX.cpp +++ lib/AST/DeclCXX.cpp @@ -2181,7 +2181,7 @@ return C.getPointerType(ClassTy); } -QualType CXXMethodDecl::getThisType(ASTContext &C) const { +QualType CXXMethodDecl::getThisType(const ASTContext &C) const { // C++ 9.3.2p1: The type of this in a member function of a class X is X*. // If the member function is declared const, the type of this is const X*, // if the member function is declared volatile, the type of this is Index: lib/AST/ExprConstant.cpp =================================================================== --- lib/AST/ExprConstant.cpp +++ lib/AST/ExprConstant.cpp @@ -9583,6 +9583,10 @@ Info.Ctx.getOpenMPDefaultSimdAlign(E->getArgumentType())) .getQuantity(), E); + case UETT_CoroFrameMaxSize: { + assert(E->isArgumentType()); + return Success(Info.Ctx.getCoroFrameMaxSize(E->getArgumentType()), E); + } } llvm_unreachable("unknown expr/type trait"); Index: lib/AST/ItaniumMangle.cpp =================================================================== --- lib/AST/ItaniumMangle.cpp +++ lib/AST/ItaniumMangle.cpp @@ -3896,7 +3896,7 @@ Diags.Report(DiagID); return; } - case UETT_OpenMPRequiredSimdAlign: + case UETT_OpenMPRequiredSimdAlign: { DiagnosticsEngine &Diags = Context.getDiags(); unsigned DiagID = Diags.getCustomDiagID( DiagnosticsEngine::Error, @@ -3904,6 +3904,15 @@ Diags.Report(DiagID); return; } + case UETT_CoroFrameMaxSize: { + DiagnosticsEngine &Diags = Context.getDiags(); + unsigned DiagID = Diags.getCustomDiagID( + DiagnosticsEngine::Error, + "cannot yet mangle __builtin_coro_frame_max_size expression"); + Diags.Report(DiagID); + return; + } + } if (SAE->isArgumentType()) { Out << 't'; mangleType(SAE->getArgumentType()); Index: lib/AST/StmtPrinter.cpp =================================================================== --- lib/AST/StmtPrinter.cpp +++ lib/AST/StmtPrinter.cpp @@ -1248,6 +1248,9 @@ case UETT_OpenMPRequiredSimdAlign: OS << "__builtin_omp_required_simd_align"; break; + case UETT_CoroFrameMaxSize: + OS << "__builtin_coro_frame_max_size"; + break; } if (Node->isArgumentType()) { OS << '('; Index: lib/Parse/ParseExpr.cpp =================================================================== --- lib/Parse/ParseExpr.cpp +++ lib/Parse/ParseExpr.cpp @@ -1185,6 +1185,8 @@ case tok::kw_vec_step: // unary-expression: OpenCL 'vec_step' expression // unary-expression: '__builtin_omp_required_simd_align' '(' type-name ')' case tok::kw___builtin_omp_required_simd_align: + // unary-exression: '__builtin_coro_frame_max_size' '(' type-name ')' + case tok::kw___builtin_coro_frame_max_size: return ParseUnaryExprOrTypeTraitExpression(); case tok::ampamp: { // unary-expression: '&&' identifier SourceLocation AmpAmpLoc = ConsumeToken(); @@ -1870,7 +1872,8 @@ assert(OpTok.isOneOf(tok::kw_typeof, tok::kw_sizeof, tok::kw___alignof, tok::kw_alignof, tok::kw__Alignof, tok::kw_vec_step, - tok::kw___builtin_omp_required_simd_align) && + tok::kw___builtin_omp_required_simd_align, + tok::kw___builtin_coro_frame_max_size) && "Not a typeof/sizeof/alignof/vec_step expression!"); ExprResult Operand; @@ -1956,7 +1959,8 @@ ExprResult Parser::ParseUnaryExprOrTypeTraitExpression() { assert(Tok.isOneOf(tok::kw_sizeof, tok::kw___alignof, tok::kw_alignof, tok::kw__Alignof, tok::kw_vec_step, - tok::kw___builtin_omp_required_simd_align) && + tok::kw___builtin_omp_required_simd_align, + tok::kw___builtin_coro_frame_max_size) && "Not a sizeof/alignof/vec_step expression!"); Token OpTok = Tok; ConsumeToken(); @@ -2032,6 +2036,8 @@ ExprKind = UETT_VecStep; else if (OpTok.is(tok::kw___builtin_omp_required_simd_align)) ExprKind = UETT_OpenMPRequiredSimdAlign; + else if (OpTok.is(tok::kw___builtin_coro_frame_max_size)) + ExprKind = UETT_CoroFrameMaxSize; if (isCastExpr) return Actions.ActOnUnaryExprOrTypeTraitExpr(OpTok.getLocation(), Index: lib/Sema/SemaExpr.cpp =================================================================== --- lib/Sema/SemaExpr.cpp +++ lib/Sema/SemaExpr.cpp @@ -3633,6 +3633,41 @@ return false; } +static bool CheckCoroFrameMaxSizeTraitOperandType(Sema &S, QualType T, + SourceLocation Loc, + SourceRange ArgRange) { + const RecordType *RTy = nullptr; + if (auto *TTy = dyn_cast(T.getTypePtr())) + RTy = dyn_cast_or_null(TTy->getReplacementType().getTypePtr()); + if (!RTy) { + S.Diag(Loc, diag::err_coro_frame_max_size_non_lambda_type) << T << ArgRange; + return true; + } + + auto *RD = dyn_cast_or_null(RTy->getDecl()); + if (!RD || !RD->isLambda()) { + S.Diag(Loc, diag::err_coro_frame_max_size_non_lambda_type) + << T << ArgRange; + return true; + } + + CXXMethodDecl *MD = RD->getLambdaCallOperator(); + if (!MD->doesThisDeclarationHaveABody()) { + S.Diag(Loc, diag::err_coro_frame_max_size_lambda_missing_body) + << T << ArgRange; + return true; + } + + auto *CoroBody = dyn_cast_or_null(MD->getBody()); + if (!CoroBody) { + S.Diag(Loc, diag::err_coro_frame_max_size_not_coroutine_lambda) + << T << ArgRange; + return true; + } + + return false; +} + static bool CheckExtensionTraitOperandType(Sema &S, QualType T, SourceLocation Loc, SourceRange ArgRange, @@ -3712,6 +3747,9 @@ if (ExprKind == UETT_VecStep) return CheckVecStepTraitOperandType(*this, ExprTy, E->getExprLoc(), E->getSourceRange()); + else if (ExprKind == UETT_CoroFrameMaxSize) + return CheckCoroFrameMaxSizeTraitOperandType(*this, ExprTy, E->getExprLoc(), + E->getSourceRange()); // Whitelist some types as extensions if (!CheckExtensionTraitOperandType(*this, ExprTy, E->getExprLoc(), @@ -3817,11 +3855,15 @@ // When alignof or _Alignof is applied to an array type, the result // is the alignment of the element type. if (ExprKind == UETT_AlignOf || ExprKind == UETT_PreferredAlignOf || - ExprKind == UETT_OpenMPRequiredSimdAlign) + ExprKind == UETT_OpenMPRequiredSimdAlign || + ExprKind == UETT_CoroFrameMaxSize) ExprType = Context.getBaseElementType(ExprType); if (ExprKind == UETT_VecStep) return CheckVecStepTraitOperandType(*this, ExprType, OpLoc, ExprRange); + else if (ExprKind == UETT_CoroFrameMaxSize) + return CheckCoroFrameMaxSizeTraitOperandType(*this, ExprType, OpLoc, + ExprRange); // Whitelist some types as extensions if (!CheckExtensionTraitOperandType(*this, ExprType, OpLoc, ExprRange, @@ -4101,8 +4143,11 @@ } else if (ExprKind == UETT_VecStep) { isInvalid = CheckVecStepExpr(E); } else if (ExprKind == UETT_OpenMPRequiredSimdAlign) { - Diag(E->getExprLoc(), diag::err_openmp_default_simd_align_expr); - isInvalid = true; + Diag(E->getExprLoc(), diag::err_openmp_default_simd_align_expr); + isInvalid = true; + } else if (ExprKind == UETT_CoroFrameMaxSize) { + Diag(E->getExprLoc(), diag::err_coro_frame_max_size_expr); + isInvalid = true; } else if (E->refersToBitField()) { // C99 6.5.3.4p1. Diag(E->getExprLoc(), diag::err_sizeof_alignof_typeof_bitfield) << 0; isInvalid = true;