diff --git a/clang/include/clang/Support/RISCVVIntrinsicUtils.h b/clang/include/clang/Support/RISCVVIntrinsicUtils.h --- a/clang/include/clang/Support/RISCVVIntrinsicUtils.h +++ b/clang/include/clang/Support/RISCVVIntrinsicUtils.h @@ -92,19 +92,54 @@ LLVM_MARK_AS_BITMASK_ENUM(LMUL1), }; -// The Lowerest two bit equal to policy value. -enum Policy : uint8_t { - TU = 0, // For unmasked TU, last two bit is TUMU - TA = 1, // // For unmasked TA, last two bit is TAMU - TUMA = 2, - TAMA = 3, - TUMU = 4, - TAMU = 5, - MU = 6, // For masked MU, last two bit is TAMU - MA = 7, // For masked MA, last two bit is TAMA - TUM = 10, // For masked MA, last two bit is TUMA - TAM = 11, // For masked MA, last two bit is TAMA - PolicyNone, +struct Policy { + bool PolicyNone = false; + bool TU = false; + bool TA = false; + bool MU = false; + bool MA = false; + bool IntrinsicWithoutMU = false; + Policy() : PolicyNone(true) {} + Policy(bool _TU, bool _TA, bool _MU, bool _MA) + : TU(_TU), TA(_TA), MU(_MU), MA(_MA) {} + Policy(bool _TU, bool _TA, bool _MU, bool _MA, bool _IntrinsicWithoutMU) + : TU(_TU), TA(_TA), MU(_MU), MA(_MA), + IntrinsicWithoutMU(_IntrinsicWithoutMU) {} + + bool isTAMAPolicy() const { return TA && MA; } + + bool isTAMUPolicy() const { return TA && MU; } + + bool isTUMAPolicy() const { return TU && MA; } + + bool isTUMUPolicy() const { return TU && MU; } + + bool isTUMPolicy() const { return TU && MA && IntrinsicWithoutMU; } + + bool isTAMPolicy() const { return TA && MA && IntrinsicWithoutMU; } + + bool isTAPolicy() const { return TA && !MA && !MU; } + + bool isTUPolicy() const { return TU && !MA && !MU; } + + bool isMAPolicy() const { return MA && !TA && !TU; } + + bool isMUPolicy() const { return MU && !TA && !TU; } + + bool isPolicyNonePolicy() const { return PolicyNone; } + + bool operator==(const Policy &Other) const { + return this->PolicyNone == Other.PolicyNone && this->TU == Other.TU && + this->TA == Other.TA && this->MU == Other.MU && this->MA == Other.MA; + } + + bool operator!=(const Policy &Other) const { return !(*this == Other); } + + bool operator<(const Policy &Other) const { + // Just for maintain the old order for quick test. + return std::tie(this->MU, this->MA, this->TA, this->TU) < + std::tie(Other.MU, Other.MA, Other.TA, Other.TU); + } }; // PrototypeDescriptor is used to compute type info of arguments or return @@ -333,7 +368,7 @@ // InputTypes. -1 means the return type. std::vector IntrinsicTypes; unsigned NF = 1; - Policy DefaultPolicy = Policy::PolicyNone; + Policy DefaultPolicy = Policy(); public: RVVIntrinsic(llvm::StringRef Name, llvm::StringRef Suffix, @@ -373,12 +408,33 @@ return IntrinsicTypes; } Policy getDefaultPolicy() const { - assert(DefaultPolicy != Policy::PolicyNone); + assert(DefaultPolicy.PolicyNone == false); return DefaultPolicy; } unsigned getDefaultPolicyBits() const { - assert(DefaultPolicy != Policy::PolicyNone); - return static_cast(DefaultPolicy) & 3; + // Return following value. + // constexpr unsigned TAIL_UNDISTURBED = 0; + // constexpr unsigned TAIL_AGNOSTIC = 1; + // constexpr unsigned TAIL_AGNOSTIC_MASK_AGNOSTIC = 3; + // FIXME: how about value 2 + // int DefaultPolicy = TAIL_UNDISTURBED; + assert(DefaultPolicy.PolicyNone == false); + + if (DefaultPolicy.isTUMAPolicy()) + return 2; + if (DefaultPolicy.isTAMAPolicy()) + return 3; + if (DefaultPolicy.isTUMUPolicy()) + return 0; + if (DefaultPolicy.isTAMUPolicy()) + return 1; + if (DefaultPolicy.isTUPolicy()) + return 0; + if (DefaultPolicy.isTAPolicy()) + return 1; + + assert(false && "unsupport policy"); + return 0; } // Return the type string for a BUILTIN() macro in Builtins.def. @@ -393,7 +449,7 @@ bool IsMasked, bool HasMaskedOffOperand, bool HasVL, unsigned NF, bool IsPrototypeDefaultTU, PolicyScheme DefaultScheme, - Policy DefaultPolicy = Policy::PolicyNone); + Policy DefaultPolicy); static llvm::SmallVector getSupportedMaskedPolicies(bool HasTailPolicy, bool HasMaskPolicy); diff --git a/clang/lib/Sema/SemaRISCVVectorLookup.cpp b/clang/lib/Sema/SemaRISCVVectorLookup.cpp --- a/clang/lib/Sema/SemaRISCVVectorLookup.cpp +++ b/clang/lib/Sema/SemaRISCVVectorLookup.cpp @@ -196,19 +196,22 @@ RVVIntrinsic::computeBuiltinTypes( BasicProtoSeq, /*IsMasked=*/false, /*HasMaskedOffOperand=*/false, Record.HasVL, Record.NF, - Record.IsPrototypeDefaultTU, UnMaskedPolicyScheme); + Record.IsPrototypeDefaultTU, UnMaskedPolicyScheme, Policy()); llvm::SmallVector ProtoMaskSeq = RVVIntrinsic::computeBuiltinTypes( BasicProtoSeq, /*IsMasked=*/true, Record.HasMaskedOffOperand, Record.HasVL, Record.NF, Record.IsPrototypeDefaultTU, - MaskedPolicyScheme); + MaskedPolicyScheme, Policy()); bool UnMaskedHasPolicy = UnMaskedPolicyScheme != PolicyScheme::SchemeNone; bool MaskedHasPolicy = MaskedPolicyScheme != PolicyScheme::SchemeNone; // If unmasked builtin supports policy, they should be TU or TA. - llvm::SmallVector SupportedUnMaskedPolicies = {Policy::TU, - Policy::TA}; + llvm::SmallVector SupportedUnMaskedPolicies; + SupportedUnMaskedPolicies.push_back( + Policy(true, false, false, false)); // TU + SupportedUnMaskedPolicies.push_back( + Policy(false, true, false, false)); // TA llvm::SmallVector SupportedMaskedPolicies = RVVIntrinsic::getSupportedMaskedPolicies(Record.HasTailPolicy, Record.HasMaskPolicy); @@ -261,7 +264,7 @@ // Create non-masked intrinsic. InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr, false, *Types, - UnMaskedHasPolicy, Policy::PolicyNone, + UnMaskedHasPolicy, Policy(), Record.IsPrototypeDefaultTU); // Create non-masked policy intrinsic. @@ -285,7 +288,7 @@ Optional MaskTypes = TypeCache.computeTypes(BaseType, Log2LMUL, Record.NF, ProtoMaskSeq); InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr, true, - *MaskTypes, MaskedHasPolicy, Policy::PolicyNone, + *MaskTypes, MaskedHasPolicy, Policy(), Record.IsPrototypeDefaultTU); if (Record.MaskedPolicyScheme == PolicyScheme::SchemeNone) continue; diff --git a/clang/lib/Support/RISCVVIntrinsicUtils.cpp b/clang/lib/Support/RISCVVIntrinsicUtils.cpp --- a/clang/lib/Support/RISCVVIntrinsicUtils.cpp +++ b/clang/lib/Support/RISCVVIntrinsicUtils.cpp @@ -914,30 +914,22 @@ SmallVector NewPrototype(Prototype.begin(), Prototype.end()); // Update DefaultPolicy if need (TA or TAMA) for compute builtin types. - switch (DefaultPolicy) { - case Policy::MA: - DefaultPolicy = Policy::TAMA; - break; - case Policy::TAM: - DefaultPolicy = Policy::TAMA; - break; - case Policy::PolicyNone: - // Masked with no policy would not be TAMA. + if (DefaultPolicy.isMAPolicy()) + DefaultPolicy.TA = true; // MA -> TAMA + if (DefaultPolicy.isPolicyNonePolicy()) { if (!IsMasked) { + DefaultPolicy.PolicyNone = false; if (IsPrototypeDefaultTU) - DefaultPolicy = Policy::TU; + DefaultPolicy.TU = true; // PolicyNone -> TU else - DefaultPolicy = Policy::TA; + DefaultPolicy.TA = true; // PolicyNone -> TA } - break; - default: - break; } bool HasPassthruOp = DefaultScheme == PolicyScheme::HasPassthruOperand; if (IsMasked) { // If HasMaskedOffOperand, insert result type as first input operand if // need. - if (HasMaskedOffOperand && DefaultPolicy != Policy::TAMA) { + if (HasMaskedOffOperand && !DefaultPolicy.isTAMAPolicy()) { if (NF == 1) { NewPrototype.insert(NewPrototype.begin() + 1, NewPrototype[0]); } else if (NF > 1) { @@ -952,7 +944,7 @@ } } // Erase passthru operand for TAM - if (NF == 1 && IsPrototypeDefaultTU && DefaultPolicy == Policy::TAMA && + if (NF == 1 && IsPrototypeDefaultTU && DefaultPolicy.isTAMAPolicy() && HasPassthruOp && !HasMaskedOffOperand) NewPrototype.erase(NewPrototype.begin() + 1); if (HasMaskedOffOperand && NF > 1) { @@ -969,21 +961,21 @@ } } else { if (NF == 1) { - if (DefaultPolicy == Policy::TU && HasPassthruOp && !IsPrototypeDefaultTU) + if (DefaultPolicy.isTUPolicy() && HasPassthruOp && !IsPrototypeDefaultTU) NewPrototype.insert(NewPrototype.begin(), NewPrototype[0]); - else if (DefaultPolicy == Policy::TA && HasPassthruOp && + else if (DefaultPolicy.isTAPolicy() && HasPassthruOp && IsPrototypeDefaultTU) NewPrototype.erase(NewPrototype.begin() + 1); if (DefaultScheme == PolicyScheme::HasPassthruOperandAtIdx1) { - if (DefaultPolicy == Policy::TU && !IsPrototypeDefaultTU) { + if (DefaultPolicy.isTUPolicy() && !IsPrototypeDefaultTU) { // Insert undisturbed output to index 1 NewPrototype.insert(NewPrototype.begin() + 2, NewPrototype[0]); - } else if (DefaultPolicy == Policy::TA && IsPrototypeDefaultTU) { + } else if (DefaultPolicy.isTAPolicy() && IsPrototypeDefaultTU) { // Erase passthru for TA policy NewPrototype.erase(NewPrototype.begin() + 2); } } - } else if (DefaultPolicy == Policy::TU && HasPassthruOp) { + } else if (DefaultPolicy.isTUPolicy() && HasPassthruOp) { // NF > 1 cases for segment load operations. // Convert // (void, op0 address, op1 address, ...) @@ -1006,10 +998,17 @@ RVVIntrinsic::getSupportedMaskedPolicies(bool HasTailPolicy, bool HasMaskPolicy) { if (HasTailPolicy && HasMaskPolicy) - return {Policy::TUMA, Policy::TAMA, Policy::TUMU, Policy::TAMU}; - else if (HasTailPolicy) - return {Policy::TUM, Policy::TAM}; - return {Policy::MA, Policy::MU}; + return {Policy(true, false, false, true), // TUMA + Policy(false, true, false, true), // TAMA + Policy(true, false, true, false), // TUMU + Policy(false, true, true, false)}; // TAMU + + if (HasTailPolicy) + return {Policy(true, false, false, true, true), // TUM + Policy(false, true, false, true, true)}; // TAM + + return {Policy(false, false, false, true), // MA + Policy(false, false, true, false)}; // MU } void RVVIntrinsic::updateNamesAndPolicy(bool IsMasked, bool HasPolicy, @@ -1025,60 +1024,51 @@ OverloadedName += suffix; }; - switch (DefaultPolicy) { - case Policy::TU: - appendPolicySuffix("_tu"); - break; - case Policy::TA: - appendPolicySuffix("_ta"); - break; - case Policy::MU: - appendPolicySuffix("_mu"); - DefaultPolicy = Policy::TAMU; - break; - case Policy::MA: - appendPolicySuffix("_ma"); - DefaultPolicy = Policy::TAMA; - break; - case Policy::TUM: - appendPolicySuffix("_tum"); - DefaultPolicy = Policy::TUMA; - break; - case Policy::TAM: - appendPolicySuffix("_tam"); - DefaultPolicy = Policy::TAMA; - break; - case Policy::TUMU: - appendPolicySuffix("_tumu"); - break; - case Policy::TAMU: - appendPolicySuffix("_tamu"); - break; - case Policy::TUMA: - appendPolicySuffix("_tuma"); - break; - case Policy::TAMA: - appendPolicySuffix("_tama"); - break; - default: + if (DefaultPolicy.isPolicyNonePolicy()) { + DefaultPolicy.PolicyNone = false; if (IsMasked) { Name += "_m"; // FIXME: Currently _m default policy implementation is different with // RVV intrinsic spec (TUMA) - DefaultPolicy = Policy::TUMU; + DefaultPolicy.TU = true; + DefaultPolicy.MU = true; if (HasPolicy) BuiltinName += "_tumu"; else BuiltinName += "_m"; } else if (IsPrototypeDefaultTU) { - DefaultPolicy = Policy::TU; + DefaultPolicy.TU = true; if (HasPolicy) BuiltinName += "_tu"; } else { - DefaultPolicy = Policy::TA; + DefaultPolicy.TA = true; if (HasPolicy) BuiltinName += "_ta"; } + } else { + if (DefaultPolicy.isTUMPolicy()) + appendPolicySuffix("_tum"); + else if (DefaultPolicy.isTAMPolicy()) + appendPolicySuffix("_tam"); + else if (DefaultPolicy.isTUMUPolicy()) + appendPolicySuffix("_tumu"); + else if (DefaultPolicy.isTAMUPolicy()) + appendPolicySuffix("_tamu"); + else if (DefaultPolicy.isTUMAPolicy()) + appendPolicySuffix("_tuma"); + else if (DefaultPolicy.isTAMAPolicy()) + appendPolicySuffix("_tama"); + else if (DefaultPolicy.isTUPolicy()) + appendPolicySuffix("_tu"); + else if (DefaultPolicy.isTAPolicy()) + appendPolicySuffix("_ta"); + else if (DefaultPolicy.isMUPolicy()) { + appendPolicySuffix("_mu"); + DefaultPolicy.TA = true; + } else if (DefaultPolicy.isMAPolicy()) { + appendPolicySuffix("_ma"); + DefaultPolicy.TA = true; + } } } diff --git a/clang/utils/TableGen/RISCVVEmitter.cpp b/clang/utils/TableGen/RISCVVEmitter.cpp --- a/clang/utils/TableGen/RISCVVEmitter.cpp +++ b/clang/utils/TableGen/RISCVVEmitter.cpp @@ -164,8 +164,7 @@ if (RVVI->getNF() >= 2) OS << " NF = " + utostr(RVVI->getNF()) + ";\n"; // We had initialized DefaultPolicy as TU/TUMU in CodeGen function. - if (RVVI->getDefaultPolicy() != Policy::TU && - RVVI->getDefaultPolicy() != Policy::TUMU && !RVVI->hasPassthruOperand() && + if (RVVI->getDefaultPolicyBits() != 0 && !RVVI->hasPassthruOperand() && !RVVI->hasManualCodegen() && RVVI->hasVL()) OS << " DefaultPolicy = " << RVVI->getDefaultPolicyBits() << ";\n"; @@ -193,11 +192,11 @@ OS << " Ops.push_back(ConstantInt::get(Ops.back()->getType()," " DefaultPolicy));\n"; if (RVVI->hasMaskedOffOperand() && - RVVI->getDefaultPolicy() == Policy::TAMA) + RVVI->getDefaultPolicy().isTAMAPolicy()) OS << " Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType));\n"; // Masked reduction cases. if (!RVVI->hasMaskedOffOperand() && RVVI->hasPassthruOperand() && - RVVI->getDefaultPolicy() == Policy::TAMA) + RVVI->getDefaultPolicy().isTAMAPolicy()) OS << " Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType));\n"; } else { OS << " std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());\n"; @@ -207,7 +206,7 @@ OS << " Ops.push_back(ConstantInt::get(Ops.back()->getType(), " "DefaultPolicy));\n"; else if (RVVI->hasPassthruOperand() && - RVVI->getDefaultPolicy() == Policy::TA) + RVVI->getDefaultPolicy().isTAPolicy()) OS << " Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType));\n"; } @@ -527,7 +526,11 @@ unsigned NF = R->getValueAsInt("NF"); // If unmasked builtin supports policy, they should be TU or TA. - SmallVector SupportedUnMaskedPolicies = {Policy::TU, Policy::TA}; + llvm::SmallVector SupportedUnMaskedPolicies; + SupportedUnMaskedPolicies.push_back( + Policy(true, false, false, false)); // TU + SupportedUnMaskedPolicies.push_back( + Policy(false, true, false, false)); // TA SmallVector SupportedMaskedPolicies = RVVIntrinsic::getSupportedMaskedPolicies(HasTailPolicy, HasMaskPolicy); @@ -544,10 +547,10 @@ auto Prototype = RVVIntrinsic::computeBuiltinTypes( BasicPrototype, /*IsMasked=*/false, /*HasMaskedOffOperand=*/false, HasVL, NF, IsPrototypeDefaultTU, - UnMaskedPolicyScheme); + UnMaskedPolicyScheme, Policy()); auto MaskedPrototype = RVVIntrinsic::computeBuiltinTypes( BasicPrototype, /*IsMasked=*/true, HasMaskedOffOperand, HasVL, NF, - IsPrototypeDefaultTU, MaskedPolicyScheme); + IsPrototypeDefaultTU, MaskedPolicyScheme, Policy()); // Create Intrinsics for each type and LMUL. for (char I : TypeRange) { @@ -569,7 +572,7 @@ /*IsMasked=*/false, /*HasMaskedOffOperand=*/false, HasVL, UnMaskedPolicyScheme, SupportOverloading, HasBuiltinAlias, ManualCodegen, *Types, IntrinsicTypes, RequiredFeatures, NF, - Policy::PolicyNone, IsPrototypeDefaultTU)); + Policy(), IsPrototypeDefaultTU)); if (UnMaskedPolicyScheme != PolicyScheme::SchemeNone) for (auto P : SupportedUnMaskedPolicies) { SmallVector PolicyPrototype = @@ -596,7 +599,7 @@ /*IsMasked=*/true, HasMaskedOffOperand, HasVL, MaskedPolicyScheme, SupportOverloading, HasBuiltinAlias, MaskedManualCodegen, MaskTypes.value(), IntrinsicTypes, RequiredFeatures, NF, - Policy::PolicyNone, IsPrototypeDefaultTU)); + Policy(), IsPrototypeDefaultTU)); if (MaskedPolicyScheme == PolicyScheme::SchemeNone) continue; for (auto P : SupportedMaskedPolicies) {