diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -2306,6 +2306,43 @@ :ref:`stackmap entry `. See the intrinsic description for further details. +.. _ob_cfp_round: + +Floating-point Rounding Mode Operand Bundles +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Rounding mode operand bundles are characterized by the ``"cfp-round"`` operand +bundle tag. They have one operand which is a metadata string. The possible +rounding mode strings are the same as for +:ref:`Constrained Floating-Point Intrinsics `. +An operation with a rounding mode operand bundle must have the `strictfp` +attribute. + +.. code-block:: llvm + + ... + + %t = call <8 x double> strictfp @llvm.vp.fadd.v8f64(<8 x double> %x, <8 x double> %y, <8 x i1> %mask, i32 %avl) [ "cfp-round"(metadata !"round.downard") ] + +.. _ob_cfp_except: + +Floating-point Exception Behavior Operand Bundles +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Exception behavior operand bundles are characterized by the ``"cfp-except"`` +operand bundle tag. They have one operand which is a metadata string. The +possible exception behavior strings are the same as for :ref:`Constrained +Floating-Point Intrinsics `. +An operation with a exception behavior operand bundle must have the `strictfp` +attribute. + + +.. code-block:: llvm + + ... + + %t = call <8 x double> strictfp @llvm.vp.fdiv.v8f64(<8 x double> %x, <8 x double> %y, <8 x i1> %mask, i32 %avl) [ "cfp-except"(metadata !"fpexcept.strict") ] + .. _moduleasm: Module-Level Inline Assembly diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -253,6 +253,19 @@ // Whether \p ID is a VP intrinsic ID. static bool IsVPIntrinsic(Intrinsic::ID); + /// Constrained FP { + + // Whether the VP intrinsic \p ID can have a rounding mode bundle. + static bool HasRoundingMode(Intrinsic::ID); + Optional getRoundingMode() const; + + // Whether the VP intrinsic \p ID can have a exception behavior + // bundle. + static bool HasExceptionBehavior(Intrinsic::ID); + Optional getExceptionBehavior() const; + + /// } Constrained FP + /// \return the mask parameter or nullptr. Value *getMaskParam() const; diff --git a/llvm/include/llvm/IR/LLVMContext.h b/llvm/include/llvm/IR/LLVMContext.h --- a/llvm/include/llvm/IR/LLVMContext.h +++ b/llvm/include/llvm/IR/LLVMContext.h @@ -93,6 +93,8 @@ OB_cfguardtarget = 3, // "cfguardtarget" OB_preallocated = 4, // "preallocated" OB_gc_live = 5, // "gc-live" + OB_cfp_round = 6, // "cfp-round" + OB_cfp_except = 7, // "cfp-except" }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -2694,9 +2694,16 @@ return true; Type *Ty = nullptr; + if (parseType(Ty)) + return true; + Value *Input = nullptr; - if (parseType(Ty) || parseValue(Ty, Input, PFS)) + if (Ty->isMetadataTy()) { + if (parseMetadataAsValue(Input, PFS)) + return true; + } else if (parseValue(Ty, Input, PFS)) return true; + Inputs.push_back(Input); } diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp --- a/llvm/lib/IR/IntrinsicInst.cpp +++ b/llvm/lib/IR/IntrinsicInst.cpp @@ -240,6 +240,35 @@ return true; } +bool VPIntrinsic::HasRoundingMode(Intrinsic::ID) { + return false; // TODO enable for fp ops. +} + +bool VPIntrinsic::HasExceptionBehavior(Intrinsic::ID) { + return false; // TODO enable for fp ops. +} + +Optional VPIntrinsic::getRoundingMode() const { + auto Bundle = this->getOperandBundle("cfp-round"); + if (!Bundle) + return None; + Metadata *MD = cast(Bundle->Inputs[0])->getMetadata(); + if (!MD || !isa(MD)) + return None; + return StrToRoundingMode(cast(MD)->getString()); +} + +Optional VPIntrinsic::getExceptionBehavior() const { + auto Bundle = this->getOperandBundle("cfp-except"); + if (!Bundle) + return None; + Metadata *MD = cast(Bundle->Inputs[0])->getMetadata(); + if (!MD || !isa(MD)) + return None; + + return StrToExceptionBehavior(cast(MD)->getString()); +} + // Equivalent non-predicated opcode unsigned VPIntrinsic::GetFunctionalOpcodeForVP(Intrinsic::ID ID) { unsigned FunctionalOC = Instruction::Call; diff --git a/llvm/lib/IR/LLVMContext.cpp b/llvm/lib/IR/LLVMContext.cpp --- a/llvm/lib/IR/LLVMContext.cpp +++ b/llvm/lib/IR/LLVMContext.cpp @@ -78,6 +78,16 @@ "gc-transition operand bundle id drifted!"); (void)GCLiveEntry; + auto *CFPRoundEntry = pImpl->getOrInsertBundleTag("cfp-round"); + assert(CFPRoundEntry->second == LLVMContext::OB_cfp_round && + "cfp-round operand bundle id drifted!"); + (void)CFPRoundEntry; + + auto *CFPExceptEntry = pImpl->getOrInsertBundleTag("cfp-except"); + assert(CFPExceptEntry->second == LLVMContext::OB_cfp_except && + "cfp-except operand bundle id drifted!"); + (void)CFPExceptEntry; + SyncScope::ID SingleThreadSSID = pImpl->getOrInsertSyncScopeID("singlethread"); assert(SingleThreadSSID == SyncScope::SingleThread && diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -75,6 +75,7 @@ #include "llvm/IR/DebugLoc.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" +#include "llvm/IR/FPEnv.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalAlias.h" #include "llvm/IR/GlobalValue.h" @@ -520,6 +521,7 @@ void verifyStatepoint(const CallBase &Call); void verifyFrameRecoverIndices(); void verifySiblingFuncletUnwinds(); + void verifyConstrainedFPBundles(const Instruction &); void verifyFragmentExpression(const DbgVariableIntrinsic &I); template @@ -2281,6 +2283,69 @@ } } +void Verifier::verifyConstrainedFPBundles(const Instruction &I) { + auto *CB = dyn_cast(&I); + if (!CB) + return; + auto ExceptBundle = CB->getOperandBundle("cfp-except"); + auto RoundBundle = CB->getOperandBundle("cfp-round"); + if (!ExceptBundle && !RoundBundle) + return; + + auto *VPIntrin = dyn_cast(&I); + Assert( + VPIntrin, + "Constrained FP bundles only enabled for Vector Predication Intrinsics", + VPIntrin); + Assert(!RoundBundle || + VPIntrinsic::HasRoundingMode(VPIntrin->getIntrinsicID()), + "Intrinsic does not accept a constrained fp rounding mode.", VPIntrin); + Assert(!ExceptBundle || + VPIntrinsic::HasExceptionBehavior(VPIntrin->getIntrinsicID()), + "Intrinsic does not accept a constrained fp exception behavior.", + VPIntrin); + + Assert(VPIntrin->hasFnAttr(Attribute::StrictFP), + "Constrained fp operation must have the 'strictfp' attribute.", + VPIntrin); + + if (RoundBundle) { + Assert(RoundBundle->Inputs.size() == 1, + "Constrained fp rounding mode bundle must have exactly one operand.", + VPIntrin); + auto &RoundInput = *RoundBundle->Inputs[0]; + Assert(isa(RoundInput), + "Constrained fp rounding mode is not a metadata string.", + RoundInput); + auto *RoundString = + dyn_cast(cast(RoundInput).getMetadata()); + Assert(RoundString, + "Constrained fp rounding mode is not a metadata string.", + RoundInput); + auto RoundOpt = VPIntrin->getRoundingMode(); + Assert(RoundOpt.hasValue(), "Invalid rounding mode metadata.", RoundString); + } + + if (ExceptBundle) { + Assert(ExceptBundle->Inputs.size() == 1, + "Constrained fp exception behavior bundle must have exactly only " + "one operand.", + VPIntrin); + auto &ExceptInput = *ExceptBundle->Inputs[0]; + Assert(isa(ExceptInput), + "Constrained fp exception behavior is not a metadata string.", + ExceptInput); + auto *ExceptString = + dyn_cast(cast(ExceptInput).getMetadata()); + Assert(ExceptString, + "Constrained fp exception behavior is not a metadata string.", + ExceptInput); + auto ExceptOpt = VPIntrin->getExceptionBehavior(); + Assert(ExceptOpt.hasValue(), "Invalid exception behavior metadata.", + ExceptString); + } +} + // visitFunction - Verify that a function is ok. // void Verifier::visitFunction(const Function &F) { @@ -3177,7 +3242,8 @@ // and at most one "preallocated" operand bundle. bool FoundDeoptBundle = false, FoundFuncletBundle = false, FoundGCTransitionBundle = false, FoundCFGuardTargetBundle = false, - FoundPreallocatedBundle = false, FoundGCLiveBundle = false;; + FoundPreallocatedBundle = false, FoundGCLiveBundle = false, + FoundCFPExceptBundle = false, FoundCFPRoundBundle = false; for (unsigned i = 0, e = Call.getNumOperandBundles(); i < e; ++i) { OperandBundleUse BU = Call.getOperandBundleAt(i); uint32_t Tag = BU.getTagID(); @@ -3218,6 +3284,13 @@ Assert(!FoundGCLiveBundle, "Multiple gc-live operand bundles", Call); FoundGCLiveBundle = true; + } else if (Tag == LLVMContext::OB_cfp_round) { + Assert(!FoundCFPRoundBundle, "Multiple cfp-round operand bundles", Call); + FoundCFPRoundBundle = true; + } else if (Tag == LLVMContext::OB_cfp_except) { + Assert(!FoundCFPExceptBundle, "Multiple cfp-except operand bundles", + Call); + FoundCFPExceptBundle = true; } } @@ -4462,6 +4535,8 @@ verifyNotEntryValue(*DII); } + verifyConstrainedFPBundles(I); + SmallVector, 4> MDs; I.getAllMetadata(MDs); for (auto Attachment : MDs) { diff --git a/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll b/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll --- a/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll +++ b/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll @@ -9,6 +9,8 @@ ; CHECK-NEXT: