Index: include/llvm/CodeGen/MachineValueType.h =================================================================== --- include/llvm/CodeGen/MachineValueType.h +++ include/llvm/CodeGen/MachineValueType.h @@ -231,6 +231,42 @@ SimpleValueType SimpleTy; + + // A class to represent the number of elements in a vector + // + // For fixed-length vectors, the total number of elements is equal to 'Min' + // For scalable vectors, the total number of elements is a multiple of 'Min' + class ElementCount { + public: + uint64_t Min; + bool Scalable; + + ElementCount(uint64_t Min, bool Scalable) + : Min(Min), Scalable(Scalable) {} + + ElementCount operator*(uint64_t RHS) { + return { Min * RHS, Scalable }; + } + + ElementCount& operator*=(uint64_t RHS) { + Min *= RHS; + return *this; + } + + ElementCount operator/(uint64_t RHS) { + return { Min / RHS, Scalable }; + } + + ElementCount& operator/=(uint64_t RHS) { + Min /= RHS; + return *this; + } + + bool operator==(const ElementCount& RHS) { + return Min == RHS.Min && Scalable == RHS.Scalable; + } + }; + constexpr MVT() : SimpleTy(INVALID_SIMPLE_VALUE_TYPE) {} constexpr MVT(SimpleValueType SVT) : SimpleTy(SVT) {} @@ -275,6 +311,15 @@ SimpleTy <= MVT::LAST_VECTOR_VALUETYPE); } + /// Return true if this is a vector value type where the + /// runtime length is machine dependent + bool isScalableVector() const { + return ((SimpleTy >= MVT::FIRST_INTEGER_SCALABLE_VALUETYPE && + SimpleTy <= MVT::LAST_INTEGER_SCALABLE_VALUETYPE) || + (SimpleTy >= MVT::FIRST_FP_SCALABLE_VALUETYPE && + SimpleTy <= MVT::LAST_FP_SCALABLE_VALUETYPE)); + } + /// Return true if this is a 16-bit vector type. bool is16BitVector() const { return (SimpleTy == MVT::v2i8 || SimpleTy == MVT::v1i16 || @@ -559,6 +604,10 @@ } } + MVT::ElementCount getVectorElementCount() const { + return { getVectorNumElements(), isScalableVector() }; + } + unsigned getSizeInBits() const { switch (SimpleTy) { default: @@ -836,6 +885,83 @@ return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE); } + static MVT getScalableVectorVT(MVT VT, unsigned NumElements) { + switch(VT.SimpleTy) { + default: + break; + case MVT::i1: + if (NumElements == 2) return MVT::nxv2i1; + if (NumElements == 4) return MVT::nxv4i1; + if (NumElements == 8) return MVT::nxv8i1; + if (NumElements == 16) return MVT::nxv16i1; + if (NumElements == 32) return MVT::nxv32i1; + break; + case MVT::i8: + if (NumElements == 1) return MVT::nxv1i8; + if (NumElements == 2) return MVT::nxv2i8; + if (NumElements == 4) return MVT::nxv4i8; + if (NumElements == 8) return MVT::nxv8i8; + if (NumElements == 16) return MVT::nxv16i8; + if (NumElements == 32) return MVT::nxv32i8; + break; + case MVT::i16: + if (NumElements == 1) return MVT::nxv1i16; + if (NumElements == 2) return MVT::nxv2i16; + if (NumElements == 4) return MVT::nxv4i16; + if (NumElements == 8) return MVT::nxv8i16; + if (NumElements == 16) return MVT::nxv16i16; + if (NumElements == 32) return MVT::nxv32i16; + break; + case MVT::i32: + if (NumElements == 1) return MVT::nxv1i32; + if (NumElements == 2) return MVT::nxv2i32; + if (NumElements == 4) return MVT::nxv4i32; + if (NumElements == 8) return MVT::nxv8i32; + if (NumElements == 16) return MVT::nxv16i32; + if (NumElements == 32) return MVT::nxv32i32; + break; + case MVT::i64: + if (NumElements == 1) return MVT::nxv1i64; + if (NumElements == 2) return MVT::nxv2i64; + if (NumElements == 4) return MVT::nxv4i64; + if (NumElements == 8) return MVT::nxv8i64; + if (NumElements == 16) return MVT::nxv16i64; + if (NumElements == 32) return MVT::nxv32i64; + break; + case MVT::f16: + if (NumElements == 2) return MVT::nxv2f16; + if (NumElements == 4) return MVT::nxv4f16; + if (NumElements == 8) return MVT::nxv8f16; + break; + case MVT::f32: + if (NumElements == 1) return MVT::nxv1f32; + if (NumElements == 2) return MVT::nxv2f32; + if (NumElements == 4) return MVT::nxv4f32; + if (NumElements == 8) return MVT::nxv8f32; + if (NumElements == 16) return MVT::nxv16f32; + break; + case MVT::f64: + if (NumElements == 1) return MVT::nxv1f64; + if (NumElements == 2) return MVT::nxv2f64; + if (NumElements == 4) return MVT::nxv4f64; + if (NumElements == 8) return MVT::nxv8f64; + break; + } + return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE); + } + + static MVT getVectorVT(MVT VT, unsigned NumElements, bool IsScalable) { + if (IsScalable) + return getScalableVectorVT(VT, NumElements); + return getVectorVT(VT, NumElements); + } + + static MVT getVectorVT(MVT VT, MVT::ElementCount EC) { + if (EC.Scalable) + return getScalableVectorVT(VT, EC.Min); + return getVectorVT(VT, EC.Min); + } + /// Return the value type corresponding to the specified type. This returns /// all pointers as iPTR. If HandleUnknown is true, unknown types are /// returned as Other, otherwise they are invalid. @@ -886,6 +1012,14 @@ MVT::FIRST_FP_VECTOR_VALUETYPE, (MVT::SimpleValueType)(MVT::LAST_FP_VECTOR_VALUETYPE + 1)); } + static mvt_range integer_wa_vector_valuetypes() { + return mvt_range(MVT::FIRST_INTEGER_SCALABLE_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_INTEGER_SCALABLE_VALUETYPE + 1)); + } + static mvt_range fp_wa_vector_valuetypes() { + return mvt_range(MVT::FIRST_FP_SCALABLE_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_FP_SCALABLE_VALUETYPE + 1)); + } /// @} }; Index: include/llvm/CodeGen/ValueTypes.h =================================================================== --- include/llvm/CodeGen/ValueTypes.h +++ include/llvm/CodeGen/ValueTypes.h @@ -67,23 +67,41 @@ /// Returns the EVT that represents a vector NumElements in length, where /// each element is of type VT. - static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements) { - MVT M = MVT::getVectorVT(VT.V, NumElements); + static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, + bool IsScalable = false) { + MVT M = MVT::getVectorVT(VT.V, NumElements, IsScalable); if (M.SimpleTy >= 0) return M; + + // We don't support extended scalable types yet. + assert(!IsScalable); return getExtendedVectorVT(Context, VT, NumElements); } + /// Returns the EVT that represents a vector EC.Min elements in length, + /// where each element is of type VT. + static EVT getVectorVT(LLVMContext &Context, EVT VT, MVT::ElementCount EC) { + MVT M = MVT::getVectorVT(VT.V, EC); + if (M.SimpleTy >= 0) + return M; + // We don't support extended scalable types yet. + assert (!EC.Scalable); + return getExtendedVectorVT(Context, VT, EC.Min); + } + /// Return a vector with the same number of elements as this vector, but /// with the element type converted to an integer type with the same /// bitwidth. EVT changeVectorElementTypeToInteger() const { + // We don't support extended scalable types yet. + assert (!isScalableVector()); if (!isSimple()) return changeExtendedVectorElementTypeToInteger(); MVT EltTy = getSimpleVT().getVectorElementType(); unsigned BitWidth = EltTy.getSizeInBits(); MVT IntTy = MVT::getIntegerVT(BitWidth); - MVT VecTy = MVT::getVectorVT(IntTy, getVectorNumElements()); + MVT VecTy = MVT::getVectorVT(IntTy, getVectorNumElements(), + isScalableVector()); assert(VecTy.SimpleTy >= 0 && "Simple vector VT not representable by simple integer vector VT!"); return VecTy; @@ -132,6 +150,15 @@ return isSimple() ? V.isVector() : isExtendedVector(); } + /// Return true if this is a vector type where the runtime + /// length is machine dependent + bool isScalableVector() const { + // We don't support extended scalable types yet. + if (!isSimple()) + return false; + return V.isScalableVector(); + } + /// Return true if this is a 16-bit vector type. bool is16BitVector() const { return isSimple() ? V.is16BitVector() : isExtended16BitVector(); @@ -247,6 +274,17 @@ return getExtendedVectorNumElements(); } + // Given a (possibly scalable) vector type, return the ElementCount + MVT::ElementCount getVectorElementCount() const { + assert((isVector()) && "Invalid vector type!"); + if (isSimple()) + return V.getVectorElementCount(); + + // We don't support extended scalable types yet. + assert(!isScalableVector()); + return {getExtendedVectorNumElements(), false}; + } + /// Return the size of the specified value type in bits. unsigned getSizeInBits() const { if (isSimple()) @@ -301,7 +339,7 @@ EVT widenIntegerVectorElementType(LLVMContext &Context) const { EVT EltVT = getVectorElementType(); EltVT = EVT::getIntegerVT(Context, 2 * EltVT.getSizeInBits()); - return EVT::getVectorVT(Context, EltVT, getVectorNumElements()); + return EVT::getVectorVT(Context, EltVT, getVectorElementCount()); } // Return a VT for an integer vector type with the size of the @@ -309,7 +347,7 @@ EVT narrowIntegerVectorElementType(LLVMContext &Context) const { EVT EltVT = getVectorElementType(); EltVT = EVT::getIntegerVT(Context, EltVT.getSizeInBits() / 2); - return EVT::getVectorVT(Context, EltVT, getVectorNumElements()); + return EVT::getVectorVT(Context, EltVT, getVectorElementCount()); } // Return a VT for a vector type with the same element type but @@ -317,7 +355,7 @@ // extended type. EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const { EVT EltVT = getVectorElementType(); - auto EltCnt = getVectorNumElements(); + auto EltCnt = getVectorElementCount(); return EVT::getVectorVT(Context, EltVT, EltCnt * 2); } @@ -326,7 +364,7 @@ // extended type. EVT getHalfNumVectorElementsVT(LLVMContext &Context) const { EVT EltVT = getVectorElementType(); - auto EltCnt = getVectorNumElements(); + auto EltCnt = getVectorElementCount(); return EVT::getVectorVT(Context, EltVT, EltCnt / 2); } @@ -342,7 +380,8 @@ if (!isPow2VectorType()) { unsigned NElts = getVectorNumElements(); unsigned Pow2NElts = 1 << Log2_32_Ceil(NElts); - return EVT::getVectorVT(Context, getVectorElementType(), Pow2NElts); + return EVT::getVectorVT(Context, getVectorElementType(), Pow2NElts, + isScalableVector()); } else { return *this; Index: lib/CodeGen/SelectionDAG/LegalizeTypes.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeTypes.cpp +++ lib/CodeGen/SelectionDAG/LegalizeTypes.cpp @@ -925,9 +925,9 @@ assert(Op.getValueType().isVector() && "Only applies to vectors!"); unsigned EltWidth = Op.getScalarValueSizeInBits(); EVT EltNVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth); - unsigned NumElts = Op.getValueType().getVectorNumElements(); + auto EltCnt = Op.getValueType().getVectorElementCount(); return DAG.getNode(ISD::BITCAST, SDLoc(Op), - EVT::getVectorVT(*DAG.getContext(), EltNVT, NumElts), Op); + EVT::getVectorVT(*DAG.getContext(), EltNVT, EltCnt), Op); } SDValue DAGTypeLegalizer::CreateStackStoreLoad(SDValue Op,