diff --git a/clang/include/clang/Support/RISCVVIntrinsicUtils.h b/clang/include/clang/Support/RISCVVIntrinsicUtils.h --- a/clang/include/clang/Support/RISCVVIntrinsicUtils.h +++ b/clang/include/clang/Support/RISCVVIntrinsicUtils.h @@ -101,12 +101,9 @@ }; PolicyType TailPolicy = Omit; PolicyType MaskPolicy = Omit; - bool IntrinsicWithoutMU = false; Policy() : IsUnspecified(true) {} - Policy(PolicyType _TailPolicy, PolicyType _MaskPolicy, - bool _IntrinsicWithoutMU = false) - : TailPolicy(_TailPolicy), MaskPolicy(_MaskPolicy), - IntrinsicWithoutMU(_IntrinsicWithoutMU) {} + Policy(PolicyType _TailPolicy, PolicyType _MaskPolicy) + : TailPolicy(_TailPolicy), MaskPolicy(_MaskPolicy) {} bool isTAMAPolicy() const { return TailPolicy == Agnostic && MaskPolicy == Agnostic; @@ -124,16 +121,6 @@ return TailPolicy == Undisturbed && MaskPolicy == Undisturbed; } - bool isTUMPolicy() const { - return TailPolicy == Undisturbed && MaskPolicy == Agnostic && - IntrinsicWithoutMU; - } - - bool isTAMPolicy() const { - return TailPolicy == Agnostic && MaskPolicy == Agnostic && - IntrinsicWithoutMU; - } - bool isTAPolicy() const { return TailPolicy == Agnostic && MaskPolicy == Omit; } @@ -154,8 +141,7 @@ bool operator==(const Policy &Other) const { return IsUnspecified == Other.IsUnspecified && - TailPolicy == Other.TailPolicy && MaskPolicy == Other.MaskPolicy && - IntrinsicWithoutMU == Other.IntrinsicWithoutMU; + TailPolicy == Other.TailPolicy && MaskPolicy == Other.MaskPolicy; } bool operator!=(const Policy &Other) const { return !(*this == Other); } @@ -382,6 +368,7 @@ bool HasMaskedOffOperand; bool HasVL; PolicyScheme Scheme; + const bool HasTailPolicy, HasMaskPolicy; bool SupportOverloading; bool HasBuiltinAlias; std::string ManualCodegen; @@ -397,7 +384,8 @@ RVVIntrinsic(llvm::StringRef Name, llvm::StringRef Suffix, llvm::StringRef OverloadedName, llvm::StringRef OverloadedSuffix, llvm::StringRef IRName, bool IsMasked, bool HasMaskedOffOperand, - bool HasVL, PolicyScheme Scheme, bool SupportOverloading, + bool HasVL, PolicyScheme Scheme, bool HasTailPolicy, + bool HasMaskPolicy, bool SupportOverloading, bool HasBuiltinAlias, llvm::StringRef ManualCodegen, const RVVTypes &Types, const std::vector &IntrinsicTypes, @@ -478,6 +466,7 @@ getSupportedMaskedPolicies(bool HasTailPolicy, bool HasMaskPolicy); static void updateNamesAndPolicy(bool IsMasked, bool HasPolicy, + bool HasTailPolicy, bool HasMaskPolicy, std::string &Name, std::string &BuiltinName, std::string &OverloadedName, Policy &PolicyAttrs); diff --git a/clang/lib/Sema/SemaRISCVVectorLookup.cpp b/clang/lib/Sema/SemaRISCVVectorLookup.cpp --- a/clang/lib/Sema/SemaRISCVVectorLookup.cpp +++ b/clang/lib/Sema/SemaRISCVVectorLookup.cpp @@ -147,7 +147,8 @@ // Create RVVIntrinsicDef. void InitRVVIntrinsic(const RVVIntrinsicRecord &Record, StringRef SuffixStr, StringRef OverloadedSuffixStr, bool IsMask, - RVVTypes &Types, bool HasPolicy, Policy PolicyAttrs); + RVVTypes &Types, bool HasPolicy, Policy PolicyAttrs, + bool HasTailPolicy, bool HasMaskPolicy); // Create FunctionDecl for a vector intrinsic. void CreateRVVIntrinsicDecl(LookupResult &LR, IdentifierInfo *II, @@ -258,7 +259,8 @@ // Create non-masked intrinsic. InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr, false, *Types, - UnMaskedHasPolicy, Policy()); + UnMaskedHasPolicy, Policy(), Record.HasTailPolicy, + Record.HasMaskPolicy); // Create non-masked policy intrinsic. if (Record.UnMaskedPolicyScheme != PolicyScheme::SchemeNone) { @@ -272,7 +274,7 @@ BaseType, Log2LMUL, Record.NF, PolicyPrototype); InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr, /*IsMask=*/false, *PolicyTypes, UnMaskedHasPolicy, - P); + P, Record.HasTailPolicy, Record.HasMaskPolicy); } } if (!Record.HasMasked) @@ -281,7 +283,8 @@ Optional MaskTypes = TypeCache.computeTypes(BaseType, Log2LMUL, Record.NF, ProtoMaskSeq); InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr, true, - *MaskTypes, MaskedHasPolicy, Policy()); + *MaskTypes, MaskedHasPolicy, Policy(), + Record.HasTailPolicy, Record.HasMaskPolicy); if (Record.MaskedPolicyScheme == PolicyScheme::SchemeNone) continue; // Create masked policy intrinsic. @@ -293,7 +296,8 @@ Optional PolicyTypes = TypeCache.computeTypes( BaseType, Log2LMUL, Record.NF, PolicyPrototype); InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr, - /*IsMask=*/true, *PolicyTypes, MaskedHasPolicy, P); + /*IsMask=*/true, *PolicyTypes, MaskedHasPolicy, P, + Record.HasTailPolicy, Record.HasMaskPolicy); } } // End for different LMUL } // End for different TypeRange @@ -304,7 +308,8 @@ void RISCVIntrinsicManagerImpl::InitRVVIntrinsic( const RVVIntrinsicRecord &Record, StringRef SuffixStr, StringRef OverloadedSuffixStr, bool IsMasked, RVVTypes &Signature, - bool HasPolicy, Policy PolicyAttrs) { + bool HasPolicy, Policy PolicyAttrs, bool HasTailPolicy, + bool HasMaskPolicy) { // Function name, e.g. vadd_vv_i32m1. std::string Name = Record.Name; if (!SuffixStr.empty()) @@ -322,7 +327,8 @@ // clang built-in function name, e.g. __builtin_rvv_vadd. std::string BuiltinName = "__builtin_rvv_" + std::string(Record.Name); - RVVIntrinsic::updateNamesAndPolicy(IsMasked, HasPolicy, Name, BuiltinName, + RVVIntrinsic::updateNamesAndPolicy(IsMasked, HasPolicy, HasTailPolicy, + HasMaskPolicy, Name, BuiltinName, OverloadedName, PolicyAttrs); // Put into IntrinsicList. diff --git a/clang/lib/Support/RISCVVIntrinsicUtils.cpp b/clang/lib/Support/RISCVVIntrinsicUtils.cpp --- a/clang/lib/Support/RISCVVIntrinsicUtils.cpp +++ b/clang/lib/Support/RISCVVIntrinsicUtils.cpp @@ -847,7 +847,8 @@ StringRef NewOverloadedName, StringRef OverloadedSuffix, StringRef IRName, bool IsMasked, bool HasMaskedOffOperand, bool HasVL, - PolicyScheme Scheme, bool SupportOverloading, + PolicyScheme Scheme, bool HasTailPolicy, + bool HasMaskPolicy, bool SupportOverloading, bool HasBuiltinAlias, StringRef ManualCodegen, const RVVTypes &OutInTypes, const std::vector &NewIntrinsicTypes, @@ -855,6 +856,7 @@ unsigned NF, Policy NewPolicyAttrs) : IRName(IRName), IsMasked(IsMasked), HasMaskedOffOperand(HasMaskedOffOperand), HasVL(HasVL), Scheme(Scheme), + HasTailPolicy(HasTailPolicy), HasMaskPolicy(HasMaskPolicy), SupportOverloading(SupportOverloading), HasBuiltinAlias(HasBuiltinAlias), ManualCodegen(ManualCodegen.str()), NF(NF), PolicyAttrs(NewPolicyAttrs) { @@ -870,8 +872,8 @@ if (!OverloadedSuffix.empty()) OverloadedName += "_" + OverloadedSuffix.str(); - updateNamesAndPolicy(IsMasked, hasPolicy(), Name, BuiltinName, OverloadedName, - PolicyAttrs); + updateNamesAndPolicy(IsMasked, hasPolicy(), HasTailPolicy, HasMaskPolicy, + Name, BuiltinName, OverloadedName, PolicyAttrs); // Init OutputType and InputTypes OutputType = OutInTypes[0]; @@ -989,9 +991,9 @@ Policy::PolicyType::Undisturbed)}; // TAMU if (HasTailPolicy && !HasMaskPolicy) return {Policy(Policy::PolicyType::Undisturbed, - Policy::PolicyType::Agnostic, true), // TUM - Policy(Policy::PolicyType::Agnostic, Policy::PolicyType::Agnostic, - true)}; // TAM + Policy::PolicyType::Agnostic), // TUM + Policy(Policy::PolicyType::Agnostic, + Policy::PolicyType::Agnostic)}; // TAM if (!HasTailPolicy && HasMaskPolicy) return { Policy(Policy::PolicyType::Omit, Policy::PolicyType::Agnostic), // MA @@ -1002,6 +1004,7 @@ } void RVVIntrinsic::updateNamesAndPolicy(bool IsMasked, bool HasPolicy, + bool HasTailPolicy, bool HasMaskPolicy, std::string &Name, std::string &BuiltinName, std::string &OverloadedName, @@ -1031,9 +1034,9 @@ BuiltinName += "_ta"; } } else { - if (PolicyAttrs.isTUMPolicy()) + if (PolicyAttrs.isTUMAPolicy() && !HasMaskPolicy) appendPolicySuffix("_tum"); - else if (PolicyAttrs.isTAMPolicy()) + else if (PolicyAttrs.isTAMAPolicy() && !HasMaskPolicy) appendPolicySuffix("_tam"); else if (PolicyAttrs.isTUMUPolicy()) appendPolicySuffix("_tumu"); @@ -1043,17 +1046,18 @@ appendPolicySuffix("_tuma"); else if (PolicyAttrs.isTAMAPolicy()) appendPolicySuffix("_tama"); - else if (PolicyAttrs.isTUPolicy()) + else if (PolicyAttrs.isTUPolicy() && !IsMasked) appendPolicySuffix("_tu"); - else if (PolicyAttrs.isTAPolicy()) + else if (PolicyAttrs.isTAPolicy() && !IsMasked) appendPolicySuffix("_ta"); - else if (PolicyAttrs.isMUPolicy()) { + else if (PolicyAttrs.isMUPolicy() && !HasTailPolicy) { appendPolicySuffix("_mu"); PolicyAttrs.TailPolicy = Policy::PolicyType::Agnostic; - } else if (PolicyAttrs.isMAPolicy()) { + } else if (PolicyAttrs.isMAPolicy() && !HasTailPolicy) { appendPolicySuffix("_ma"); PolicyAttrs.TailPolicy = Policy::PolicyType::Agnostic; - } + } else + llvm_unreachable("Unhandled policy condition"); } } diff --git a/clang/utils/TableGen/RISCVVEmitter.cpp b/clang/utils/TableGen/RISCVVEmitter.cpp --- a/clang/utils/TableGen/RISCVVEmitter.cpp +++ b/clang/utils/TableGen/RISCVVEmitter.cpp @@ -569,9 +569,9 @@ Out.push_back(std::make_unique( Name, SuffixStr, OverloadedName, OverloadedSuffixStr, IRName, /*IsMasked=*/false, /*HasMaskedOffOperand=*/false, HasVL, - UnMaskedPolicyScheme, SupportOverloading, HasBuiltinAlias, - ManualCodegen, *Types, IntrinsicTypes, RequiredFeatures, NF, - Policy())); + UnMaskedPolicyScheme, HasTailPolicy, HasMaskPolicy, + SupportOverloading, HasBuiltinAlias, ManualCodegen, *Types, + IntrinsicTypes, RequiredFeatures, NF, Policy())); if (UnMaskedPolicyScheme != PolicyScheme::SchemeNone) for (auto P : SupportedUnMaskedPolicies) { SmallVector PolicyPrototype = @@ -584,9 +584,9 @@ Out.push_back(std::make_unique( Name, SuffixStr, OverloadedName, OverloadedSuffixStr, IRName, /*IsMask=*/false, /*HasMaskedOffOperand=*/false, HasVL, - UnMaskedPolicyScheme, SupportOverloading, HasBuiltinAlias, - ManualCodegen, *PolicyTypes, IntrinsicTypes, RequiredFeatures, - NF, P)); + UnMaskedPolicyScheme, HasTailPolicy, HasMaskPolicy, + SupportOverloading, HasBuiltinAlias, ManualCodegen, + *PolicyTypes, IntrinsicTypes, RequiredFeatures, NF, P)); } if (!HasMasked) continue; @@ -596,8 +596,9 @@ Out.push_back(std::make_unique( Name, SuffixStr, OverloadedName, OverloadedSuffixStr, MaskedIRName, /*IsMasked=*/true, HasMaskedOffOperand, HasVL, MaskedPolicyScheme, - SupportOverloading, HasBuiltinAlias, ManualCodegen, *MaskTypes, - IntrinsicTypes, RequiredFeatures, NF, Policy())); + HasTailPolicy, HasMaskPolicy, SupportOverloading, HasBuiltinAlias, + ManualCodegen, *MaskTypes, IntrinsicTypes, RequiredFeatures, NF, + Policy())); if (MaskedPolicyScheme == PolicyScheme::SchemeNone) continue; for (auto P : SupportedMaskedPolicies) { @@ -610,9 +611,9 @@ Out.push_back(std::make_unique( Name, SuffixStr, OverloadedName, OverloadedSuffixStr, MaskedIRName, /*IsMasked=*/true, HasMaskedOffOperand, HasVL, - MaskedPolicyScheme, SupportOverloading, HasBuiltinAlias, - ManualCodegen, *PolicyTypes, IntrinsicTypes, RequiredFeatures, NF, - P)); + MaskedPolicyScheme, HasTailPolicy, HasMaskPolicy, + SupportOverloading, HasBuiltinAlias, ManualCodegen, *PolicyTypes, + IntrinsicTypes, RequiredFeatures, NF, P)); } } // End for Log2LMULList } // End for TypeRange