diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -1056,6 +1056,26 @@ form and the known alignment of the pointer specified to the call site. If the alignment is not specified, then the code generator makes a target-specific assumption. +``preallocated()`` + This indicates that the pointer parameter should really be passed by + value to the function, and that the pointer parameter's pointee has + already been initialized before the call instruction. This attribute + is only valid on LLVM pointer arguments. The argument must be the value + returned by the appropriate + :ref:`llvm.call.preallocated.arg`, although is + ignored during codegen. + + Any function call with a ``preallocated`` attribute in any parameter + must have a ``"preallocated"`` operand bundle. + + The preallocated attribute requires a type argument, which must be + the same as the pointee type of the argument. + + The preallocated attribute also supports specifying an alignment with the + align attribute. It indicates the alignment of the stack slot to + form and the known alignment of the pointer specified to the call + site. If the alignment is not specified, then the code generator + makes a target-specific assumption. .. _attr_inalloca: @@ -1953,6 +1973,12 @@ := optional, custom name of the vector function +``preallocated()`` + This attribute is required on calls to ``llvm.call.preallocated.arg`` + and cannot be used on any other call. See + :ref:`llvm.call.preallocated.arg` for more + details. + .. _glattrs: Global Attributes @@ -2165,6 +2191,33 @@ simplifies and improves heuristics, e.g., for use "use-sensitive" optimizations. +.. _ob_preallocated: + +Preallocated Operand Bundles +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Preallocated operand bundles are characterized by the ``"preallocated"`` +operand bundle tag. These operand bundles allow separation of the allocation +of the call argument memory from the call site. This is necessary to pass +non-trivially copyable objects by value in a way that is compatible with MSVC +on some targets. There can be at most one ``"preallocated"`` operand bundle +attached to a call site and it must have exactly one bundle operand, which is +a token generated by ``@llvm.call.preallocated.setup``. A call with this +operand bundle should not adjust the stack before entering the function, as +that will have been done by one of the ``@llvm.call.preallocated.*`` intrinsics. + +.. code-block:: llvm + + %foo = type { i64, i32 } + + ... + + %t = call token @llvm.call.preallocated.setup(i32 1) + %a = call i8* @llvm.call.preallocated.arg(token %t, i32 0) preallocated(%foo) + %b = bitcast i8* %a to %foo* + ; initialize %b + call void @bar(i32 42, %foo* preallocated(%foo) %b) ["preallocated"(token %t)] + .. _moduleasm: Module-Level Inline Assembly @@ -11874,6 +11927,90 @@ other operations necessary to locate the TLS area. Not all targets support this intrinsic. +'``llvm.call.preallocated.setup``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare token @llvm.call.preallocated.setup(i32 %num_args) + +Overview: +""""""""" + +The '``llvm.call.preallocated.setup``' intrinsic returns a token which can +be used with a call's ``"preallocated"`` operand bundle to indicate that +certain arguments are allocated and initialized before the call. + +Semantics: +"""""""""" + +The '``llvm.call.preallocated.setup``' intrinsic returns a token which is +associated with at most one call. The token can be passed to +'``@llvm.call.preallocated.arg``' to get a pointer to get that +corresponding argument. The token must be the parameter to a +``"preallocated"`` operand bundle for the corresponding call. + +Nested calls to '``llvm.call.preallocated.setup``' are allowed, but must +be properly nested. e.g. + +:: code-block:: llvm + + %t1 = call token @llvm.call.preallocated.setup(i32 0) + %t2 = call token @llvm.call.preallocated.setup(i32 0) + call void foo() ["preallocated"(token %t2)] + call void foo() ["preallocated"(token %t1)] + +is allowed, but not + +:: code-block:: llvm + + %t1 = call token @llvm.call.preallocated.setup(i32 0) + %t2 = call token @llvm.call.preallocated.setup(i32 0) + call void foo() ["preallocated"(token %t1)] + call void foo() ["preallocated"(token %t2)] + +.. _int_call_preallocated_arg: + +'``llvm.call.preallocated.arg``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare i8* @llvm.call.preallocated.arg(token %setup_token, i32 %arg_index) + +Overview: +""""""""" + +The '``llvm.call.preallocated.arg``' intrinsic returns a pointer to the +corresponding preallocated argument for the preallocated call. + +Semantics: +"""""""""" + +The '``llvm.call.preallocated.arg``' intrinsic returns a pointer to the +``%arg_index``th argument with the ``preallocated`` attribute for +the call associated with the ``%setup_token``, which must be from +'``llvm.call.preallocated.setup``'. + +A call to '``llvm.call.preallocated.arg``' must have a call site +``preallocated`` attribute. The type of the ``preallocated`` attribute must +match the type used by the ``preallocated`` attribute of the corresponding +argument at the preallocated call. The type is used in the case that an +``llvm.call.preallocated.setup`` does not have a corresponding call (e.g. due +to DCE), where otherwise we cannot know how large the arguments are. + +It is undefined behavior if this is called with a token from an +'``llvm.call.preallocated.setup``' if another +'``llvm.call.preallocated.setup``' has already been called or if the +preallocated call corresponding to the '``llvm.call.preallocated.setup``' +has already been called. + Standard C Library Intrinsics ----------------------------- diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -633,6 +633,7 @@ ATTR_KIND_NOFREE = 62, ATTR_KIND_NOSYNC = 63, ATTR_KIND_SANITIZE_MEMTAG = 64, + ATTR_KIND_PREALLOCATED = 65, }; enum ComdatSelectionKindCodes { diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h --- a/llvm/include/llvm/IR/Attributes.h +++ b/llvm/include/llvm/IR/Attributes.h @@ -108,6 +108,7 @@ unsigned ElemSizeArg, const Optional &NumElemsArg); static Attribute getWithByValType(LLVMContext &Context, Type *Ty); + static Attribute getWithPreallocatedType(LLVMContext &Context, Type *Ty); static Attribute::AttrKind getAttrKindFromName(StringRef AttrName); @@ -302,6 +303,7 @@ uint64_t getDereferenceableBytes() const; uint64_t getDereferenceableOrNullBytes() const; Type *getByValType() const; + Type *getPreallocatedType() const; std::pair> getAllocSizeArgs() const; std::string getAsString(bool InAttrGrp = false) const; @@ -724,6 +726,7 @@ uint64_t DerefOrNullBytes = 0; uint64_t AllocSizeArgs = 0; Type *ByValType = nullptr; + Type *PreallocatedType = nullptr; public: AttrBuilder() = default; @@ -802,6 +805,9 @@ /// Retrieve the byval type. Type *getByValType() const { return ByValType; } + /// Retrieve the preallocated type. + Type *getPreallocatedType() const { return PreallocatedType; } + /// Retrieve the allocsize args, if the allocsize attribute exists. If it /// doesn't exist, pair(0, 0) is returned. std::pair> getAllocSizeArgs() const; @@ -845,6 +851,9 @@ /// This turns a byval type into the form used internally in Attribute. AttrBuilder &addByValAttr(Type *Ty); + /// This turns a preallocated type into the form used internally in Attribute. + AttrBuilder &addPreallocatedAttr(Type *Ty); + /// Add an allocsize attribute, using the representation returned by /// Attribute.getIntValue(). AttrBuilder &addAllocSizeAttrFromRawRepr(uint64_t RawAllocSizeRepr); diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td --- a/llvm/include/llvm/IR/Attributes.td +++ b/llvm/include/llvm/IR/Attributes.td @@ -133,6 +133,9 @@ /// Function must not be optimized. def OptimizeNone : EnumAttr<"optnone">; +/// Similar to byval but without a copy. +def Preallocated : TypeAttr<"preallocated">; + /// Function does not access memory. def ReadNone : EnumAttr<"readnone">; diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -508,6 +508,9 @@ llvm_i32_ty], []>; +def int_call_preallocated_setup : Intrinsic<[llvm_token_ty], [llvm_i32_ty]>; +def int_call_preallocated_arg : Intrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_i32_ty]>; + //===------------------- Standard C Library Intrinsics --------------------===// // diff --git a/llvm/include/llvm/IR/LLVMContext.h b/llvm/include/llvm/IR/LLVMContext.h --- a/llvm/include/llvm/IR/LLVMContext.h +++ b/llvm/include/llvm/IR/LLVMContext.h @@ -83,12 +83,14 @@ /// Known operand bundle tag IDs, which always have the same value. All /// operand bundle tags that LLVM has special knowledge of are listed here. /// Additionally, this scheme allows LLVM to efficiently check for specific - /// operand bundle tags without comparing strings. + /// operand bundle tags without comparing strings. Keep this in sync with + /// LLVMContext::LLVMContext(). enum : unsigned { OB_deopt = 0, // "deopt" OB_funclet = 1, // "funclet" OB_gc_transition = 2, // "gc-transition" OB_cfguardtarget = 3, // "cfguardtarget" + OB_preallocated = 4, // "preallocated" }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp --- a/llvm/lib/AsmParser/LLLexer.cpp +++ b/llvm/lib/AsmParser/LLLexer.cpp @@ -667,6 +667,7 @@ KEYWORD(optforfuzzing); KEYWORD(optnone); KEYWORD(optsize); + KEYWORD(preallocated); KEYWORD(readnone); KEYWORD(readonly); KEYWORD(returned); diff --git a/llvm/lib/AsmParser/LLParser.h b/llvm/lib/AsmParser/LLParser.h --- a/llvm/lib/AsmParser/LLParser.h +++ b/llvm/lib/AsmParser/LLParser.h @@ -338,6 +338,7 @@ std::vector &FwdRefAttrGrps, bool inAttrGrp, LocTy &BuiltinLoc); bool ParseByValWithOptionalType(Type *&Result); + bool ParsePreallocated(Type *&Result); // Module Summary Index Parsing. bool SkipModuleSummaryEntry(); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -1345,6 +1345,13 @@ case lltok::kw_uwtable: B.addAttribute(Attribute::UWTable); break; case lltok::kw_willreturn: B.addAttribute(Attribute::WillReturn); break; case lltok::kw_writeonly: B.addAttribute(Attribute::WriteOnly); break; + case lltok::kw_preallocated: { + Type *Ty; + if (ParsePreallocated(Ty)) + return true; + B.addPreallocatedAttr(Ty); + break; + } // Error handling. case lltok::kw_inreg: @@ -1373,7 +1380,9 @@ break; } - Lex.Lex(); + // ParsePreallocated() consumes token + if (Token != lltok::kw_preallocated) + Lex.Lex(); } } @@ -1637,6 +1646,13 @@ B.addByValAttr(Ty); continue; } + case lltok::kw_preallocated: { + Type *Ty; + if (ParsePreallocated(Ty)) + return true; + B.addPreallocatedAttr(Ty); + continue; + } case lltok::kw_dereferenceable: { uint64_t Bytes; if (ParseOptionalDerefAttrBytes(lltok::kw_dereferenceable, Bytes)) @@ -1804,10 +1820,15 @@ case lltok::kw_uwtable: HaveError |= Error(Lex.getLoc(), "invalid use of function-only attribute"); break; - case lltok::kw_readnone: case lltok::kw_readonly: HaveError |= Error(Lex.getLoc(), "invalid use of attribute on return type"); + break; + case lltok::kw_preallocated: + HaveError |= + Error(Lex.getLoc(), + "invalid use of parameter-only/call site-only attribute"); + break; } Lex.Lex(); @@ -2519,6 +2540,21 @@ return false; } +/// ParsePreallocated +/// ::= preallocated() +bool LLParser::ParsePreallocated(Type *&Result) { + Result = nullptr; + if (!EatIfPresent(lltok::kw_preallocated)) + return true; + if (!EatIfPresent(lltok::lparen)) + return Error(Lex.getLoc(), "expected '('"); + if (ParseType(Result)) + return true; + if (!EatIfPresent(lltok::rparen)) + return Error(Lex.getLoc(), "expected ')'"); + return false; +} + /// ParseOptionalOperandBundles /// ::= /*empty*/ /// ::= '[' OperandBundle [, OperandBundle ]* ']' diff --git a/llvm/lib/AsmParser/LLToken.h b/llvm/lib/AsmParser/LLToken.h --- a/llvm/lib/AsmParser/LLToken.h +++ b/llvm/lib/AsmParser/LLToken.h @@ -213,6 +213,7 @@ kw_optforfuzzing, kw_optnone, kw_optsize, + kw_preallocated, kw_readnone, kw_readonly, kw_returned, diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -1303,6 +1303,9 @@ case Attribute::SanitizeMemTag: llvm_unreachable("sanitize_memtag attribute not supported in raw format"); break; + case Attribute::Preallocated: + llvm_unreachable("preallocated attribute not supported in raw format"); + break; } llvm_unreachable("Unsupported attribute type"); } @@ -1312,12 +1315,10 @@ for (Attribute::AttrKind I = Attribute::None; I != Attribute::EndAttrKinds; I = Attribute::AttrKind(I + 1)) { - if (I == Attribute::SanitizeMemTag || - I == Attribute::Dereferenceable || - I == Attribute::DereferenceableOrNull || - I == Attribute::ArgMemOnly || - I == Attribute::AllocSize || - I == Attribute::NoSync) + if (I == Attribute::SanitizeMemTag || I == Attribute::Dereferenceable || + I == Attribute::DereferenceableOrNull || I == Attribute::ArgMemOnly || + I == Attribute::AllocSize || I == Attribute::NoSync || + I == Attribute::Preallocated) continue; if (uint64_t A = (Val & getRawAttributeMask(I))) { if (I == Attribute::Alignment) @@ -1544,6 +1545,8 @@ return Attribute::ImmArg; case bitc::ATTR_KIND_SANITIZE_MEMTAG: return Attribute::SanitizeMemTag; + case bitc::ATTR_KIND_PREALLOCATED: + return Attribute::Preallocated; } } @@ -1659,8 +1662,11 @@ Attribute::AttrKind Kind; if (Error Err = parseAttrKind(Record[++i], &Kind)) return Err; - if (Kind == Attribute::ByVal) + if (Kind == Attribute::ByVal) { B.addByValAttr(HasType ? getTypeByID(Record[++i]) : nullptr); + } else if (Kind == Attribute::Preallocated) { + B.addPreallocatedAttr(getTypeByID(Record[++i])); + } } } diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -725,6 +725,8 @@ return bitc::ATTR_KIND_IMMARG; case Attribute::SanitizeMemTag: return bitc::ATTR_KIND_SANITIZE_MEMTAG; + case Attribute::Preallocated: + return bitc::ATTR_KIND_PREALLOCATED; case Attribute::EndAttrKinds: llvm_unreachable("Can not encode end-attribute kinds marker."); case Attribute::None: diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -4197,9 +4197,15 @@ return; } - assert(Attr.hasAttribute(Attribute::ByVal) && "unexpected type attr"); + assert(Attr.hasAttribute(Attribute::ByVal) || + Attr.hasAttribute(Attribute::Preallocated) && "unexpected type attr"); + + if (Attr.hasAttribute(Attribute::ByVal)) { + Out << "byval"; + } else { + Out << "preallocated"; + } - Out << "byval"; if (Type *Ty = Attr.getValueAsType()) { Out << '('; TypePrinter.print(Ty, Out); diff --git a/llvm/lib/IR/AttributeImpl.h b/llvm/lib/IR/AttributeImpl.h --- a/llvm/lib/IR/AttributeImpl.h +++ b/llvm/lib/IR/AttributeImpl.h @@ -220,6 +220,7 @@ std::pair> getAllocSizeArgs() const; std::string getAsString(bool InAttrGrp) const; Type *getByValType() const; + Type *getPreallocatedType() const; using iterator = const Attribute *; diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp --- a/llvm/lib/IR/Attributes.cpp +++ b/llvm/lib/IR/Attributes.cpp @@ -169,6 +169,10 @@ return get(Context, ByVal, Ty); } +Attribute Attribute::getWithPreallocatedType(LLVMContext &Context, Type *Ty) { + return get(Context, Preallocated, Ty); +} + Attribute Attribute::getWithAllocSizeArgs(LLVMContext &Context, unsigned ElemSizeArg, const Optional &NumElemsArg) { @@ -446,6 +450,17 @@ return Result; } + if (hasAttribute(Attribute::Preallocated)) { + std::string Result; + Result += "preallocated"; + raw_string_ostream OS(Result); + Result += '('; + getValueAsType()->print(OS, false, true); + OS.flush(); + Result += ')'; + return Result; + } + // FIXME: These should be output like this: // // align=4 @@ -731,6 +746,10 @@ return SetNode ? SetNode->getByValType() : nullptr; } +Type *AttributeSet::getPreallocatedType() const { + return SetNode ? SetNode->getPreallocatedType() : nullptr; +} + std::pair> AttributeSet::getAllocSizeArgs() const { return SetNode ? SetNode->getAllocSizeArgs() : std::pair>(0, 0); @@ -829,6 +848,9 @@ case Attribute::ByVal: Attr = Attribute::getWithByValType(C, B.getByValType()); break; + case Attribute::Preallocated: + Attr = Attribute::getWithPreallocatedType(C, B.getPreallocatedType()); + break; case Attribute::Alignment: assert(B.getAlignment() && "Alignment must be set"); Attr = Attribute::getWithAlignment(C, *B.getAlignment()); @@ -912,6 +934,13 @@ return 0; } +Type *AttributeSetNode::getPreallocatedType() const { + for (const auto &I : *this) + if (I.hasAttribute(Attribute::Preallocated)) + return I.getValueAsType(); + return 0; +} + uint64_t AttributeSetNode::getDereferenceableBytes() const { if (auto A = findEnumAttribute(Attribute::Dereferenceable)) return A->getDereferenceableBytes(); @@ -1495,6 +1524,7 @@ DerefBytes = DerefOrNullBytes = 0; AllocSizeArgs = 0; ByValType = nullptr; + PreallocatedType = nullptr; } AttrBuilder &AttrBuilder::addAttribute(Attribute::AttrKind Val) { @@ -1520,6 +1550,8 @@ StackAlignment = Attr.getStackAlignment(); else if (Kind == Attribute::ByVal) ByValType = Attr.getValueAsType(); + else if (Kind == Attribute::Preallocated) + PreallocatedType = Attr.getValueAsType(); else if (Kind == Attribute::Dereferenceable) DerefBytes = Attr.getDereferenceableBytes(); else if (Kind == Attribute::DereferenceableOrNull) @@ -1544,6 +1576,8 @@ StackAlignment.reset(); else if (Val == Attribute::ByVal) ByValType = nullptr; + else if (Val == Attribute::Preallocated) + PreallocatedType = nullptr; else if (Val == Attribute::Dereferenceable) DerefBytes = 0; else if (Val == Attribute::DereferenceableOrNull) @@ -1632,6 +1666,12 @@ return *this; } +AttrBuilder &AttrBuilder::addPreallocatedAttr(Type *Ty) { + Attrs[Attribute::Preallocated] = true; + PreallocatedType = Ty; + return *this; +} + AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) { // FIXME: What if both have alignments, but they don't match?! if (!Alignment) @@ -1652,6 +1692,9 @@ if (!ByValType) ByValType = B.ByValType; + if (!PreallocatedType) + PreallocatedType = B.PreallocatedType; + Attrs |= B.Attrs; for (auto I : B.td_attrs()) @@ -1680,6 +1723,9 @@ if (B.ByValType) ByValType = nullptr; + if (B.PreallocatedType) + PreallocatedType = nullptr; + Attrs &= ~B.Attrs; for (auto I : B.td_attrs()) @@ -1739,7 +1785,8 @@ return false; return Alignment == B.Alignment && StackAlignment == B.StackAlignment && - DerefBytes == B.DerefBytes && ByValType == B.ByValType; + DerefBytes == B.DerefBytes && ByValType == B.ByValType && + PreallocatedType == B.PreallocatedType; } //===----------------------------------------------------------------------===// @@ -1757,17 +1804,18 @@ if (!Ty->isPointerTy()) // Attribute that only apply to pointers. - Incompatible.addAttribute(Attribute::ByVal) - .addAttribute(Attribute::Nest) - .addAttribute(Attribute::NoAlias) - .addAttribute(Attribute::NoCapture) - .addAttribute(Attribute::NonNull) - .addDereferenceableAttr(1) // the int here is ignored - .addDereferenceableOrNullAttr(1) // the int here is ignored - .addAttribute(Attribute::ReadNone) - .addAttribute(Attribute::ReadOnly) - .addAttribute(Attribute::StructRet) - .addAttribute(Attribute::InAlloca); + Incompatible.addAttribute(Attribute::Nest) + .addAttribute(Attribute::NoAlias) + .addAttribute(Attribute::NoCapture) + .addAttribute(Attribute::NonNull) + .addDereferenceableAttr(1) // the int here is ignored + .addDereferenceableOrNullAttr(1) // the int here is ignored + .addAttribute(Attribute::ReadNone) + .addAttribute(Attribute::ReadOnly) + .addAttribute(Attribute::StructRet) + .addAttribute(Attribute::InAlloca) + .addPreallocatedAttr(Ty) + .addByValAttr(Ty); return Incompatible; } diff --git a/llvm/lib/IR/LLVMContext.cpp b/llvm/lib/IR/LLVMContext.cpp --- a/llvm/lib/IR/LLVMContext.cpp +++ b/llvm/lib/IR/LLVMContext.cpp @@ -68,6 +68,11 @@ "cfguardtarget operand bundle id drifted!"); (void)CFGuardTargetEntry; + auto *PreallocatedEntry = pImpl->getOrInsertBundleTag("preallocated"); + assert(PreallocatedEntry->second == LLVMContext::OB_preallocated && + "preallocated operand bundle id drifted!"); + (void)PreallocatedEntry; + SyncScope::ID SingleThreadSSID = pImpl->getOrInsertSyncScopeID("singlethread"); assert(SingleThreadSSID == SyncScope::SingleThread && diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -1563,7 +1563,8 @@ /// arguments. static bool isFuncOrArgAttr(Attribute::AttrKind Kind) { return Kind == Attribute::ReadOnly || Kind == Attribute::WriteOnly || - Kind == Attribute::ReadNone || Kind == Attribute::NoFree; + Kind == Attribute::ReadNone || Kind == Attribute::NoFree || + Kind == Attribute::Preallocated; } void Verifier::verifyAttributeTypes(AttributeSet Attrs, bool IsFunction, @@ -1614,11 +1615,13 @@ unsigned AttrCount = 0; AttrCount += Attrs.hasAttribute(Attribute::ByVal); AttrCount += Attrs.hasAttribute(Attribute::InAlloca); + AttrCount += Attrs.hasAttribute(Attribute::Preallocated); AttrCount += Attrs.hasAttribute(Attribute::StructRet) || Attrs.hasAttribute(Attribute::InReg); AttrCount += Attrs.hasAttribute(Attribute::Nest); - Assert(AttrCount <= 1, "Attributes 'byval', 'inalloca', 'inreg', 'nest', " - "and 'sret' are incompatible!", + Assert(AttrCount <= 1, + "Attributes 'byval', 'inalloca', 'preallocated', 'inreg', 'nest', " + "and 'sret' are incompatible!", V); Assert(!(Attrs.hasAttribute(Attribute::InAlloca) && @@ -1668,6 +1671,12 @@ "Attribute 'byval' type does not match parameter!", V); } + if (Attrs.hasAttribute(Attribute::Preallocated)) { + Assert(Attrs.getPreallocatedType() == + cast(Ty)->getElementType(), + "Attribute 'preallocated' type does not match parameter!", V); + } + AttrBuilder IncompatibleAttrs = AttributeFuncs::typeIncompatible(Ty); Assert(!AttrBuilder(Attrs).overlaps(IncompatibleAttrs), "Wrong types for attribute: " + @@ -1678,8 +1687,10 @@ SmallPtrSet Visited; if (!PTy->getElementType()->isSized(&Visited)) { Assert(!Attrs.hasAttribute(Attribute::ByVal) && - !Attrs.hasAttribute(Attribute::InAlloca), - "Attributes 'byval' and 'inalloca' do not support unsized types!", + !Attrs.hasAttribute(Attribute::InAlloca) && + !Attrs.hasAttribute(Attribute::Preallocated), + "Attributes 'byval', 'inalloca', and 'preallocated' do not " + "support unsized types!", V); } if (!isa(PTy->getElementType())) @@ -1720,9 +1731,11 @@ !RetAttrs.hasAttribute(Attribute::NoFree) && !RetAttrs.hasAttribute(Attribute::Returned) && !RetAttrs.hasAttribute(Attribute::InAlloca) && + !RetAttrs.hasAttribute(Attribute::Preallocated) && !RetAttrs.hasAttribute(Attribute::SwiftSelf) && !RetAttrs.hasAttribute(Attribute::SwiftError)), - "Attributes 'byval', 'inalloca', 'nest', 'sret', 'nocapture', 'nofree'" + "Attributes 'byval', 'inalloca', 'preallocated', 'nest', 'sret', " + "'nocapture', 'nofree', " "'returned', 'swiftself', and 'swifterror' do not apply to return " "values!", V); @@ -2904,6 +2917,14 @@ "speculatable attribute may not apply to call sites", Call); } + if (Attrs.hasAttribute(AttributeList::FunctionIndex, + Attribute::Preallocated)) { + Assert(Call.getCalledFunction()->getIntrinsicID() == + Intrinsic::call_preallocated_arg, + "preallocated as a call site attribute can only be on " + "llvm.call.preallocated.arg"); + } + // Verify call attributes. verifyFunctionAttrs(FTy, Attrs, &Call, IsIntrinsic); @@ -2950,6 +2971,13 @@ Assert(isa(ArgVal) || isa(ArgVal), "immarg operand has non-immediate parameter", ArgVal, Call); } + + if (Call.paramHasAttr(i, Attribute::Preallocated)) { + Value *ArgVal = Call.getArgOperand(i); + Assert(Call.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0, + "preallocated operand requires a preallocated bundle", ArgVal, + Call); + } } if (FTy->isVarArg()) { @@ -3020,9 +3048,11 @@ visitIntrinsicCall(ID, Call); // Verify that a callsite has at most one "deopt", at most one "funclet", at - // most one "gc-transition", and at most one "cfguardtarget" operand bundle. + // most one "gc-transition", at most one "cfguardtarget", + // and at most one "preallocated" operand bundle. bool FoundDeoptBundle = false, FoundFuncletBundle = false, - FoundGCTransitionBundle = false, FoundCFGuardTargetBundle = false; + FoundGCTransitionBundle = false, FoundCFGuardTargetBundle = false, + FoundPreallocatedBundle = false; for (unsigned i = 0, e = Call.getNumOperandBundles(); i < e; ++i) { OperandBundleUse BU = Call.getOperandBundleAt(i); uint32_t Tag = BU.getTagID(); @@ -3047,6 +3077,18 @@ FoundCFGuardTargetBundle = true; Assert(BU.Inputs.size() == 1, "Expected exactly one cfguardtarget bundle operand", Call); + } else if (Tag == LLVMContext::OB_preallocated) { + Assert(!FoundPreallocatedBundle, "Multiple preallocated operand bundles", + Call); + FoundPreallocatedBundle = true; + Assert(BU.Inputs.size() == 1, + "Expected exactly one preallocated bundle operand", Call); + auto Input = dyn_cast(BU.Inputs.front()); + Assert(Input && + Input->getIntrinsicID() == Intrinsic::call_preallocated_setup, + "\"preallocated\" argument must be a token from " + "llvm.call.preallocated.setup", + Call); } } @@ -3077,8 +3119,9 @@ static AttrBuilder getParameterABIAttributes(int I, AttributeList Attrs) { static const Attribute::AttrKind ABIAttrs[] = { - Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca, - Attribute::InReg, Attribute::SwiftSelf, Attribute::SwiftError}; + Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca, + Attribute::InReg, Attribute::SwiftSelf, Attribute::SwiftError, + Attribute::Preallocated}; AttrBuilder Copy; for (auto AK : ABIAttrs) { if (Attrs.hasParamAttribute(I, AK)) @@ -3120,7 +3163,7 @@ "cannot guarantee tail call due to mismatched calling conv", &CI); // - All ABI-impacting function attributes, such as sret, byval, inreg, - // returned, and inalloca, must match. + // returned, preallocated, and inalloca, must match. AttributeList CallerAttrs = F->getAttributes(); AttributeList CalleeAttrs = CI.getAttributes(); for (int I = 0, E = CallerTy->getNumParams(); I != E; ++I) { @@ -4432,6 +4475,70 @@ } break; } + case Intrinsic::call_preallocated_setup: { + auto *NumArgs = dyn_cast(Call.getArgOperand(0)); + Assert(NumArgs != nullptr, + "llvm.call.preallocated.setup argument must be a constant"); + bool FoundCall = false; + for (User *U : Call.users()) { + auto *UseCall = dyn_cast(U); + Assert(UseCall != nullptr, + "Uses of llvm.call.preallocated.setup must be calls"); + const Function *Fn = UseCall->getCalledFunction(); + if (Fn->getIntrinsicID() == Intrinsic::call_preallocated_arg) { + auto *AllocArgIndex = dyn_cast(UseCall->getArgOperand(1)); + Assert(AllocArgIndex != nullptr, + "llvm.call.preallocated.alloc arg index must be a constant"); + auto AllocArgIndexInt = AllocArgIndex->getValue(); + Assert(AllocArgIndexInt.sge(0) && + AllocArgIndexInt.slt(NumArgs->getValue()), + "llvm.call.preallocated.alloc arg index must be between 0 and " + "corresponding " + "llvm.call.preallocated.setup's argument count"); + } else { + Assert(!FoundCall, "Can have at most one call corresponding to a " + "llvm.call.preallocated.setup"); + FoundCall = true; + size_t NumPreallocatedArgs = 0; + for (auto &Arg : Fn->args()) { + if (Arg.hasAttribute(Attribute::Preallocated)) { + ++NumPreallocatedArgs; + } + } + Assert(NumArgs->equalsInt(NumPreallocatedArgs), + "llvm.call.preallocated.setup arg size must be equal to number " + "of arguments " + "at call site"); + // getOperandBundle() cannot be called if more than one of the operand + // bundle exists. There is already a check elsewhere for this, so skip + // here if we see more than one. + if (UseCall->countOperandBundlesOfType(LLVMContext::OB_preallocated) > + 1) { + return; + } + auto PreallocatedBundle = + UseCall->getOperandBundle(LLVMContext::OB_preallocated); + Assert(PreallocatedBundle, + "Use of llvm.call.preallocated.setup outside intrinsics " + "must be in \"preallocated\" operand bundle"); + Assert(PreallocatedBundle->Inputs.front().get() == &Call, + "preallocated bundle must have token from corresponding " + "llvm.call.preallocated.setup"); + } + } + break; + } + case Intrinsic::call_preallocated_arg: { + auto *Token = dyn_cast(Call.getArgOperand(0)); + Assert(Token && Token->getCalledFunction()->getIntrinsicID() == + Intrinsic::call_preallocated_setup, + "llvm.call.preallocated.arg token argument must be a " + "llvm.call.preallocated.setup"); + Assert(Call.hasFnAttr(Attribute::Preallocated), + "llvm.call.preallocated.arg must be called with a \"preallocated\" " + "call site attribute"); + break; + } case Intrinsic::gcroot: case Intrinsic::gcwrite: case Intrinsic::gcread: diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp --- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp +++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp @@ -878,6 +878,7 @@ case Attribute::NoSync: case Attribute::None: case Attribute::NonNull: + case Attribute::Preallocated: case Attribute::ReadNone: case Attribute::ReadOnly: case Attribute::Returned: diff --git a/llvm/test/Assembler/invalid-byval-type3.ll b/llvm/test/Assembler/invalid-byval-type3.ll --- a/llvm/test/Assembler/invalid-byval-type3.ll +++ b/llvm/test/Assembler/invalid-byval-type3.ll @@ -1,4 +1,4 @@ ; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s -; CHECK: Attributes 'byval' and 'inalloca' do not support unsized types! +; CHECK: Attributes 'byval'{{.*}} do not support unsized types! declare void @foo(void()* byval(void())) diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll --- a/llvm/test/Bitcode/attributes.ll +++ b/llvm/test/Bitcode/attributes.ll @@ -371,7 +371,13 @@ ; CHECK: define void @f63() #39 define void @f63() sanitize_memtag { - ret void; + ret void +} + +; CHECK: define void @f64(i32* preallocated(i32) %a) +define void @f64(i32* preallocated(i32) %a) +{ + ret void } ; CHECK: attributes #0 = { noreturn } diff --git a/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll b/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll --- a/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll +++ b/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll @@ -7,6 +7,7 @@ ; CHECK-NEXT: &1 | FileCheck %s + +declare token @llvm.call.preallocated.setup(i32) +declare i8* @llvm.call.preallocated.arg(token, i32) + +; Fake LLVM intrinsic to return a token +declare token @llvm.what() + +declare void @foo0() +declare void @foo1(i32* preallocated(i32)) +declare void @foo2(i32* preallocated(i32), i32*, i32* preallocated(i32)) +declare i32 @blackbox() + +; CHECK: llvm.call.preallocated.arg must be called with a "preallocated" call site attribute +define void @preallocated_arg_missing_preallocated_attribute() { + %cs = call token @llvm.call.preallocated.setup(i32 1) + %x = call i8* @llvm.call.preallocated.arg(token %cs, i32 0) + %y = bitcast i8* %x to i32* + call void @foo1(i32* preallocated(i32) %y) ["preallocated"(token %cs)] + ret void +} + +; CHECK: preallocated as a call site attribute can only be on llvm.call.preallocated.arg +define void @preallocated_call_site_attribute_not_on_arg() { + call void @foo0() preallocated(i32) + ret void +} + +; CHECK: "preallocated" argument must be a token from llvm.call.preallocated.setup +define void @preallocated_bundle_token() { + %i = call i32 @blackbox() + call void @foo0() ["preallocated"(i32 %i)] + ret void +} + +; CHECK: "preallocated" argument must be a token from llvm.call.preallocated.setup +define void @preallocated_bundle_token_from_setup() { + %cs = call token @llvm.what() + call void @foo0() ["preallocated"(token %cs)] + ret void +} + +; CHECK: Expected exactly one preallocated bundle operand +define void @preallocated_bundle_one_token() { + %cs0 = call token @llvm.call.preallocated.setup(i32 0) + %cs1 = call token @llvm.call.preallocated.setup(i32 0) + call void @foo0() ["preallocated"(token %cs0, token %cs1)] + ret void +} + +; CHECK: Multiple preallocated operand bundles +define void @preallocated_multiple_bundles() { + %cs0 = call token @llvm.call.preallocated.setup(i32 0) + %cs1 = call token @llvm.call.preallocated.setup(i32 0) + call void @foo0() ["preallocated"(token %cs0), "preallocated"(token %cs1)] + ret void +} + +; CHECK: Can have at most one call +define void @preallocated_one_call() { + %cs = call token @llvm.call.preallocated.setup(i32 1) + %x = call i8* @llvm.call.preallocated.arg(token %cs, i32 0) preallocated(i32) + %y = bitcast i8* %x to i32* + call void @foo1(i32* preallocated(i32) %y) ["preallocated"(token %cs)] + call void @foo1(i32* preallocated(i32) %y) ["preallocated"(token %cs)] + ret void +} + +; CHECK: must be a constant +define void @preallocated_setup_constant() { + %ac = call i32 @blackbox() + %cs = call token @llvm.call.preallocated.setup(i32 %ac) + ret void +} + +; CHECK: must be between 0 and corresponding +define void @preallocated_setup_arg_index_in_bounds() { + %cs = call token @llvm.call.preallocated.setup(i32 2) + %a0 = call i8* @llvm.call.preallocated.arg(token %cs, i32 2) preallocated(i32) + ret void +} + +; CHECK: Attribute 'preallocated' type does not match parameter +define void @preallocated_attribute_type_mismatch() { + %cs = call token @llvm.call.preallocated.setup(i32 1) + %x = call i8* @llvm.call.preallocated.arg(token %cs, i32 0) preallocated(i32) + %y = bitcast i8* %x to i32* + call void @foo1(i32* preallocated(i8) %y) ["preallocated"(token %cs)] + ret void +} + +; CHECK: preallocated operand requires a preallocated bundle +define void @preallocated_require_bundle() { + %cs = call token @llvm.call.preallocated.setup(i32 1) + %x = call i8* @llvm.call.preallocated.arg(token %cs, i32 0) preallocated(i32) + %y = bitcast i8* %x to i32* + call void @foo1(i32* preallocated(i32) %y) + ret void +} + +; CHECK: arg size must be equal to number of arguments +define void @preallocated_num_args() { + %cs = call token @llvm.call.preallocated.setup(i32 3) + %x = call i8* @llvm.call.preallocated.arg(token %cs, i32 0) preallocated(i32) + %x1 = bitcast i8* %x to i32* + %y = call i8* @llvm.call.preallocated.arg(token %cs, i32 1) preallocated(i32) + %y1 = bitcast i8* %y to i32* + %a = inttoptr i32 0 to i32* + call void @foo2(i32* preallocated(i32) %x1, i32* %a, i32* preallocated(i32) %y1) ["preallocated"(token %cs)] + ret void +} + +; CHECK: token argument must be a llvm.call.preallocated.setup +define void @preallocated_arg_token() { + %t = call token @llvm.what() + %x = call i8* @llvm.call.preallocated.arg(token %t, i32 1) preallocated(i32) + ret void +} diff --git a/llvm/test/Verifier/preallocated-valid.ll b/llvm/test/Verifier/preallocated-valid.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Verifier/preallocated-valid.ll @@ -0,0 +1,32 @@ +; RUN: opt -S %s -verify + +declare token @llvm.call.preallocated.setup(i32) +declare i8* @llvm.call.preallocated.arg(token, i32) + +declare void @foo1(i32* preallocated(i32)) +declare void @foo2(i32* preallocated(i32), i32*, i32* preallocated(i32)) + +define void @preallocated() { + %cs = call token @llvm.call.preallocated.setup(i32 1) + %x = call i8* @llvm.call.preallocated.arg(token %cs, i32 0) preallocated(i32) + %y = bitcast i8* %x to i32* + call void @foo1(i32* preallocated(i32) %y) ["preallocated"(token %cs)] + ret void +} + +define void @preallocated_setup_without_call() { + %cs = call token @llvm.call.preallocated.setup(i32 1) + %a0 = call i8* @llvm.call.preallocated.arg(token %cs, i32 0) preallocated(i32) + ret void +} + +define void @preallocated_num_args() { + %cs = call token @llvm.call.preallocated.setup(i32 2) + %x = call i8* @llvm.call.preallocated.arg(token %cs, i32 0) preallocated(i32) + %x1 = bitcast i8* %x to i32* + %y = call i8* @llvm.call.preallocated.arg(token %cs, i32 1) preallocated(i32) + %y1 = bitcast i8* %y to i32* + %a = inttoptr i32 0 to i32* + call void @foo2(i32* preallocated(i32) %x1, i32* %a, i32* preallocated(i32) %y1) ["preallocated"(token %cs)] + ret void +}