Index: llvm/include/llvm/Support/TypeSize.h =================================================================== --- llvm/include/llvm/Support/TypeSize.h +++ llvm/include/llvm/Support/TypeSize.h @@ -15,69 +15,207 @@ #ifndef LLVM_SUPPORT_TYPESIZE_H #define LLVM_SUPPORT_TYPESIZE_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/WithColor.h" -#include #include +#include +#include namespace llvm { template struct DenseMapInfo; -// TODO: This class will be redesigned in a later patch that introduces full -// polynomial behaviour, i.e. the ability to have composites made up of both -// fixed and scalable sizes. -template class PolySize { +/// PolyBase is a base class for ElementCount, TypeSize and StackOffset. +/// It tries to represent a true polynomial, e.g. +/// c0 * scale0 + c1 * scale1 + ... + cK * scaleK +/// where the scale is implicit. Only the coefficients are encoded. +/// +/// is the type of the coefficients. +/// are the number of dimensions of the polynomial. +/// lets PolyBase represent a polynomial with coefficients +/// for only a single dimension when true, or multiple coefficients when false. +template class PolyBase { protected: - T MinVal; // The minimum value that it could be. - bool IsScalable; // If true, the total value is determined by multiplying - // 'MinVal' by a runtime determinded quantity, 'vscale'. + T Coefficients[Dims]; + unsigned ExclusiveDim; + + // Create a PolyBase for multiple dimensions. + template + PolyBase(ArrayRef::type> Values) + : ExclusiveDim(~0) { + assert(Values.size() == Dims && "Incorrect number of values"); + for (unsigned I = 0; I < Dims; ++I) + Coefficients[I] = Values[I]; + verify(); + } - constexpr PolySize(T MinVal, bool IsScalable) - : MinVal(MinVal), IsScalable(IsScalable) {} + // Create a PolyBase for a single dimension. + template + PolyBase(const typename std::enable_if<(D), T>::type &Val, + unsigned ExclusiveDim) + : ExclusiveDim(ExclusiveDim) { + assert(ExclusiveDim < Dims && "Incorrect dimension"); + for (unsigned I = 0; I < Dims; ++I) + Coefficients[I] = (I == ExclusiveDim ? Val : 0); + verify(); + } + + void verify() const { +#ifndef NDEBUG + assert((!DimIsExclusive || + llvm::count_if(ArrayRef(Coefficients, Dims), + [](const T &C) { return C > 0; }) <= 1) && + "More than one dimensions set for Mutually Exclusive PolyBase"); +#endif + } public: + bool operator==(const PolyBase &RHS) const { + return std::equal(&Coefficients[0], &Coefficients[Dims], + &RHS.Coefficients[0]); + } + + bool operator!=(const PolyBase &RHS) const { return !(*this == RHS); } + + PolyBase &operator+=(const PolyBase &RHS) { + for (unsigned I = 0; I < Dims; ++I) + Coefficients[I] += RHS.Coefficients[I]; + verify(); + return *this; + } + + PolyBase &operator-=(const PolyBase &RHS) { + for (unsigned I = 0; I < Dims; ++I) + Coefficients[I] -= RHS.Coefficients[I]; + verify(); + return *this; + } + + PolyBase &operator*=(T RHS) { + for (unsigned I = 0; I < Dims; ++I) + Coefficients[I] *= RHS; + return *this; + } + + PolyBase operator+(const PolyBase &RHS) const { + PolyBase This = *this; + return This += RHS; + } + + PolyBase operator-(const PolyBase &RHS) const { + PolyBase This = *this; + return This -= RHS; + } + + PolyBase operator*(T RHS) const { + PolyBase This = *this; + return This *= RHS; + } + + template + typename std::enable_if::value, PolyBase>::type + operator-() const { + PolyBase This = *this; + return This *= (T)-1; + } - static constexpr PolySize getFixed(T MinVal) { return {MinVal, false}; } - static constexpr PolySize getScalable(T MinVal) { return {MinVal, true}; } - static constexpr PolySize get(T MinVal, bool IsScalable) { - return {MinVal, IsScalable}; + bool isZero() const { + return llvm::all_of(ArrayRef(Coefficients, Dims), + [](const T &C) { return C == 0; }); } - static constexpr PolySize getNull() { return {0, false}; } + bool isNonZero() const { return !isZero(); } - /// Counting predicates. - /// - ///@{ No elements.. - bool isZero() const { return MinVal == 0; } - /// At least one element. - bool isNonZero() const { return MinVal != 0; } + explicit operator bool() const { return isNonZero(); } + + T getValue(unsigned Dim) const { return Coefficients[Dim]; } + + template + typename std::enable_if<(D), T>::type getExclusiveValue() const { + return Coefficients[ExclusiveDim]; + } +}; + + +template +using MixedPolyBase2DSuper_t = + PolyBase; + +/// MixedPolyBase2D is a base class to represent 2 dimensions, named +/// fixed and scalable, respectively. This class allows a value for both +/// dimensions to depict e.g. "8 bytes and 16 scalable bytes", which is needed +/// to represent stack offsets. +template class MixedPolyBase2D : public MixedPolyBase2DSuper_t { +protected: + MixedPolyBase2D(T Fixed, T Scalable) + : MixedPolyBase2DSuper_t({Fixed, Scalable}) {} + +public: + MixedPolyBase2D() : MixedPolyBase2DSuper_t({0, 0}) {} + MixedPolyBase2D(const MixedPolyBase2DSuper_t &Other) + : MixedPolyBase2DSuper_t(Other) {} + static MixedPolyBase2D getFixed(T Fixed) { return {Fixed, 0}; } + static MixedPolyBase2D getScalable(T Scalable) { return {0, Scalable}; } + static MixedPolyBase2D get(T Fixed, T Scalable) { return {Fixed, Scalable}; } + + T getFixed() const { return this->getValue(0); } + T getScalable() const { return this->getValue(1); } + std::pair getMixed() const { + return {this->getValue(0), this->getValue(1)}; + } +}; + +template +using SinglePolyBase2DSuper_t = + PolyBase; + +/// SinglePolyBase2D is a base class to represent 2 dimensions, where the +/// base class can only represent 1 dimension exclusively at a time, i.e. it is +/// either fixed-sized or it is scalable-sized, but it cannot be both. +template +class SinglePolyBase2D : public SinglePolyBase2DSuper_t { +protected: + SinglePolyBase2D(T MinVal, bool IsScalable) + : SinglePolyBase2DSuper_t(MinVal, !IsScalable ? 0 : 1) {} + +public: + SinglePolyBase2D(const SinglePolyBase2DSuper_t &Other) + : SinglePolyBase2DSuper_t(Other) {} + static SinglePolyBase2D getFixed(T MinVal) { return {MinVal, false}; } + static SinglePolyBase2D getScalable(T MinVal) { return {MinVal, true}; } + static SinglePolyBase2D get(T MinVal, bool Scalable) { + return {MinVal, Scalable}; + } + + /// Returns the minimum value this size can represent. + T getKnownMinValue() const { return this->getExclusiveValue(); } + /// Returns whether the size is scaled by a runtime quantity (vscale). + bool isScalable() const { return this->ExclusiveDim == 1; } /// A return value of true indicates we know at compile time that the number /// of elements (vscale * Min) is definitely even. However, returning false /// does not guarantee that the total number of elements is odd. - bool isKnownEven() const { return (MinVal & 0x1) == 0; } - ///@} - - T getKnownMinValue() const { return MinVal; } + bool isKnownEven() const { return (getKnownMinValue() & 0x1) == 0; } + /// This function tells the caller whether the element count is known at + /// compile time to be a multiple of the scalar value RHS. + bool isKnownMultipleOf(T RHS) const { return getKnownMinValue() % RHS == 0; } + /// Exactly one element. + bool isScalar() const { return !isScalable() && getKnownMinValue() == 1; } + /// One or more elements. + bool isVector() const { + return (isScalable() && getKnownMinValue() != 0) || getKnownMinValue() > 1; + } // Return the minimum value with the assumption that the count is exact. // Use in places where a scalable count doesn't make sense (e.g. non-vector // types, or vectors in backends which don't support scalable vectors). T getFixedValue() const { - assert(!IsScalable && + assert(!isScalable() && "Request for a fixed element count on a scalable object"); - return MinVal; - } - - bool isScalable() const { return IsScalable; } - - bool operator==(const PolySize &RHS) const { - return MinVal == RHS.MinVal && IsScalable == RHS.IsScalable; + return getKnownMinValue(); } - bool operator!=(const PolySize &RHS) const { return !(*this == RHS); } - // For some cases, size ordering between scalable and fixed size types cannot // be determined at compile time, so such comparisons aren't allowed. // @@ -88,55 +226,34 @@ // All the functions below make use of the fact vscale is always >= 1, which // means that is guaranteed to be >= <4 x i32>, etc. - static bool isKnownLT(const PolySize &LHS, const PolySize &RHS) { - if (!LHS.IsScalable || RHS.IsScalable) - return LHS.MinVal < RHS.MinVal; - - // LHS.IsScalable = true, RHS.IsScalable = false + static bool isKnownLT(const SinglePolyBase2D &LHS, + const SinglePolyBase2D &RHS) { + if (!LHS.isScalable() || RHS.isScalable()) + return LHS.getKnownMinValue() < RHS.getKnownMinValue(); return false; } - static bool isKnownGT(const PolySize &LHS, const PolySize &RHS) { - if (LHS.IsScalable || !RHS.IsScalable) - return LHS.MinVal > RHS.MinVal; - - // LHS.IsScalable = false, RHS.IsScalable = true + static bool isKnownGT(const SinglePolyBase2D &LHS, + const SinglePolyBase2D &RHS) { + if (LHS.isScalable() || !RHS.isScalable()) + return LHS.getKnownMinValue() > RHS.getKnownMinValue(); return false; } - static bool isKnownLE(const PolySize &LHS, const PolySize &RHS) { - if (!LHS.IsScalable || RHS.IsScalable) - return LHS.MinVal <= RHS.MinVal; - - // LHS.IsScalable = true, RHS.IsScalable = false + static bool isKnownLE(const SinglePolyBase2D &LHS, + const SinglePolyBase2D &RHS) { + if (!LHS.isScalable() || RHS.isScalable()) + return LHS.getKnownMinValue() <= RHS.getKnownMinValue(); return false; } - static bool isKnownGE(const PolySize &LHS, const PolySize &RHS) { - if (LHS.IsScalable || !RHS.IsScalable) - return LHS.MinVal >= RHS.MinVal; - - // LHS.IsScalable = false, RHS.IsScalable = true + static bool isKnownGE(const SinglePolyBase2D &LHS, + const SinglePolyBase2D &RHS) { + if (LHS.isScalable() || !RHS.isScalable()) + return LHS.getKnownMinValue() >= RHS.getKnownMinValue(); return false; } - PolySize operator*(T RHS) { return {MinVal * RHS, IsScalable}; } - - PolySize &operator*=(T RHS) { - MinVal *= RHS; - return *this; - } - - friend PolySize operator-(const PolySize &LHS, const PolySize &RHS) { - assert(LHS.IsScalable == RHS.IsScalable && - "Arithmetic using mixed scalable and fixed types"); - return {LHS.MinVal - RHS.MinVal, LHS.IsScalable}; - } - - /// This function tells the caller whether the element count is known at - /// compile time to be a multiple of the scalar value RHS. - bool isKnownMultipleOf(T RHS) const { return MinVal % RHS == 0; } - /// We do not provide the '/' operator here because division for polynomial /// types does not work in the same way as for normal integer types. We can /// only divide the minimum value (or coefficient) by RHS, which is not the @@ -145,72 +262,60 @@ /// The caller is recommended to use this function in combination with /// isKnownMultipleOf(RHS), which lets the caller know if it's possible to /// perform a lossless divide by RHS. - PolySize divideCoefficientBy(T RHS) const { - return PolySize(MinVal / RHS, IsScalable); + SinglePolyBase2D divideCoefficientBy(T RHS) const { + return SinglePolyBase2D(getKnownMinValue() / RHS, isScalable()); } - PolySize coefficientNextPowerOf2() const { - return PolySize((T)llvm::NextPowerOf2(MinVal), IsScalable); + SinglePolyBase2D coefficientNextPowerOf2() const { + return SinglePolyBase2D((T)llvm::NextPowerOf2(getKnownMinValue()), + isScalable()); } /// Printing function. void print(raw_ostream &OS) const { - if (IsScalable) + if (isScalable()) OS << "vscale x "; - OS << MinVal; + OS << getKnownMinValue(); } }; -/// Stream operator function for `PolySize`. +/// Stream operator function for `SinglePolyBase2D`. template -inline raw_ostream &operator<<(raw_ostream &OS, const PolySize &PS) { +inline raw_ostream &operator<<(raw_ostream &OS, const SinglePolyBase2D &PS) { PS.print(OS); return OS; } -class ElementCount : public PolySize { -public: - - constexpr ElementCount(PolySize V) : PolySize(V) {} +using ElementCount = SinglePolyBase2D; +using StackOffset = MixedPolyBase2D; - /// Counting predicates. - /// - /// Notice that MinVal = 1 and IsScalable = true is considered more than - /// one element. - /// - ///@{ No elements.. - /// Exactly one element. - bool isScalar() const { return !IsScalable && MinVal == 1; } - /// One or more elements. - bool isVector() const { return (IsScalable && MinVal != 0) || MinVal > 1; } - ///@} -}; - -// This class is used to represent the size of types. If the type is of fixed +// TODO: Most functionality in this class will gradually be phased out +// so it will resemble SinglePolyBase2D as much as possible. +// +// TypeSize is used to represent the size of types. If the type is of fixed // size, it will represent the exact size. If the type is a scalable vector, // it will represent the known minimum size. -class TypeSize : public PolySize { +class TypeSize : public SinglePolyBase2D { public: - constexpr TypeSize(PolySize V) : PolySize(V) {} + TypeSize(const SinglePolyBase2D &V) : SinglePolyBase2D(V) {} - constexpr TypeSize(uint64_t MinVal, bool IsScalable) - : PolySize(MinVal, IsScalable) {} + TypeSize(uint64_t MinVal, bool IsScalable) + : SinglePolyBase2D(MinVal, IsScalable) {} - static constexpr TypeSize Fixed(uint64_t MinVal) { - return TypeSize(MinVal, false); - } - static constexpr TypeSize Scalable(uint64_t MinVal) { - return TypeSize(MinVal, true); - } + static TypeSize Fixed(uint64_t MinVal) { return TypeSize(MinVal, false); } + static TypeSize Scalable(uint64_t MinVal) { return TypeSize(MinVal, true); } uint64_t getFixedSize() const { return getFixedValue(); } uint64_t getKnownMinSize() const { return getKnownMinValue(); } + // The comparison operators are in the process of being phased out + // in favour of isKnownLT/isKnownLE of its parent class. + friend bool operator<(const TypeSize &LHS, const TypeSize &RHS) { - assert(LHS.IsScalable == RHS.IsScalable && + assert(LHS.isScalable() == RHS.isScalable() && "Ordering comparison of scalable and fixed types"); - return LHS.MinVal < RHS.MinVal; + return LHS.getKnownMinValue() < RHS.getKnownMinValue(); } friend bool operator>(const TypeSize &LHS, const TypeSize &RHS) { @@ -221,29 +326,15 @@ return !(RHS < LHS); } - friend bool operator>=(const TypeSize &LHS, const TypeSize& RHS) { + friend bool operator>=(const TypeSize &LHS, const TypeSize &RHS) { return !(LHS < RHS); } - TypeSize &operator-=(TypeSize RHS) { - assert(IsScalable == RHS.IsScalable && - "Subtraction using mixed scalable and fixed types"); - MinVal -= RHS.MinVal; - return *this; - } - - TypeSize &operator+=(TypeSize RHS) { - assert(IsScalable == RHS.IsScalable && - "Addition using mixed scalable and fixed types"); - MinVal += RHS.MinVal; - return *this; - } - - friend TypeSize operator-(const TypeSize &LHS, const TypeSize &RHS) { - assert(LHS.IsScalable == RHS.IsScalable && - "Arithmetic using mixed scalable and fixed types"); - return {LHS.MinVal - RHS.MinVal, LHS.IsScalable}; - } + // All code for this class below this point is needed because of the + // temporary implicit conversion to uint64_t. The operator overloads are + // needed because otherwise the conversion of the parent class PolyBase -> + // TypeSize is ambiguous. + // TODO: Remove the implicit conversion. // Casts to a uint64_t if this is a fixed-width size. // @@ -275,32 +366,49 @@ #endif } - // Convenience operators to obtain relative sizes independently of - // the scalable flag. - TypeSize operator*(unsigned RHS) const { return {MinVal * RHS, IsScalable}; } - - friend TypeSize operator*(const unsigned LHS, const TypeSize &RHS) { - return {LHS * RHS.MinVal, RHS.IsScalable}; + // Overload the parent operators. + TypeSize &operator+=(const TypeSize &RHS) { + return (TypeSize &)SinglePolyBase2D::operator+=(RHS); + } + TypeSize &operator-=(const TypeSize &RHS) { + return (TypeSize &)SinglePolyBase2D::operator-=(RHS); + } + TypeSize &operator*=(const uint64_t &RHS) { + return (TypeSize &)SinglePolyBase2D::operator*=(RHS); + } + TypeSize operator+(const TypeSize &RHS) const { + return (TypeSize)SinglePolyBase2D::operator+(RHS); + } + TypeSize operator-(const TypeSize &RHS) const { + return (TypeSize)SinglePolyBase2D::operator-(RHS); + } + TypeSize operator*(const uint64_t &RHS) const { + return (TypeSize)SinglePolyBase2D::operator*(RHS); } - // Additional convenience operators needed to avoid ambiguous parses. - // TODO: Make uint64_t the default operator? - TypeSize operator*(uint64_t RHS) const { return {MinVal * RHS, IsScalable}; } - - TypeSize operator*(int RHS) const { return {MinVal * RHS, IsScalable}; } - - TypeSize operator*(int64_t RHS) const { return {MinVal * RHS, IsScalable}; } - - friend TypeSize operator*(const uint64_t LHS, const TypeSize &RHS) { - return {LHS * RHS.MinVal, RHS.IsScalable}; + // Additional operators needed to avoid ambiguous parses + // because of the implicit conversion hack. + TypeSize operator*(const unsigned &RHS) const { + return (TypeSize)SinglePolyBase2D::operator*(RHS); + } + TypeSize operator*(const int &RHS) const { + return (TypeSize)SinglePolyBase2D::operator*(RHS); + } + TypeSize operator*(const int64_t &RHS) const { + return (TypeSize)SinglePolyBase2D::operator*(RHS); } friend TypeSize operator*(const int LHS, const TypeSize &RHS) { - return {LHS * RHS.MinVal, RHS.IsScalable}; + return RHS * LHS; + } + friend TypeSize operator*(const unsigned LHS, const TypeSize &RHS) { + return RHS * LHS; } - friend TypeSize operator*(const int64_t LHS, const TypeSize &RHS) { - return {LHS * RHS.MinVal, RHS.IsScalable}; + return RHS * LHS; + } + friend TypeSize operator*(const uint64_t LHS, const TypeSize &RHS) { + return RHS * LHS; } }; @@ -322,7 +430,7 @@ static inline ElementCount getTombstoneKey() { return ElementCount::getFixed(~0U - 1); } - static unsigned getHashValue(const ElementCount& EltCnt) { + static unsigned getHashValue(const ElementCount &EltCnt) { unsigned HashVal = EltCnt.getKnownMinValue() * 37U; if (EltCnt.isScalable()) return (HashVal - 1U); @@ -330,7 +438,7 @@ return HashVal; } - static bool isEqual(const ElementCount& LHS, const ElementCount& RHS) { + static bool isEqual(const ElementCount &LHS, const ElementCount &RHS) { return LHS == RHS; } }; Index: mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h =================================================================== --- mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h +++ mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h @@ -17,7 +17,9 @@ #include "mlir/IR/Types.h" namespace llvm { -class ElementCount; +template +class SinglePolyBase2D; +using ElementCount = SinglePolyBase2D; class TypeSize; } // namespace llvm