diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -427,7 +427,7 @@ virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const { // The default action for one element vectors is to scalarize - if (VT.getVectorElementCount() == 1) + if (VT.getVectorElementCount().isScalar()) return TypeScalarizeVector; // The default action for an odd-width vector is to widen. if (!VT.isPow2VectorType()) diff --git a/llvm/include/llvm/CodeGen/ValueTypes.h b/llvm/include/llvm/CodeGen/ValueTypes.h --- a/llvm/include/llvm/CodeGen/ValueTypes.h +++ b/llvm/include/llvm/CodeGen/ValueTypes.h @@ -214,9 +214,7 @@ } /// Return true if the bit size is a multiple of 8. - bool isByteSized() const { - return getSizeInBits().isByteSized(); - } + bool isByteSized() const { return getSizeInBits().isKnownMultipleOf(8); } /// Return true if the size is a power-of-two number of bytes. bool isRound() const { diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h --- a/llvm/include/llvm/IR/Intrinsics.h +++ b/llvm/include/llvm/IR/Intrinsics.h @@ -188,8 +188,7 @@ } static IITDescriptor getVector(unsigned Width, bool IsScalable) { - IITDescriptor Result; - Result.Kind = Vector; + IITDescriptor Result = {Vector, 0}; Result.Vector_Width = ElementCount::get(Width, IsScalable); return Result; } diff --git a/llvm/include/llvm/Support/MachineValueType.h b/llvm/include/llvm/Support/MachineValueType.h --- a/llvm/include/llvm/Support/MachineValueType.h +++ b/llvm/include/llvm/Support/MachineValueType.h @@ -980,9 +980,7 @@ /// Returns true if the number of bits for the type is a multiple of an /// 8-bit byte. - bool isByteSized() const { - return getSizeInBits().isByteSized(); - } + bool isByteSized() const { return getSizeInBits().isKnownMultipleOf(8); } /// Return true if we know at compile time this has more bits than VT. bool knownBitsGT(MVT VT) const { diff --git a/llvm/include/llvm/Support/TypeSize.h b/llvm/include/llvm/Support/TypeSize.h --- a/llvm/include/llvm/Support/TypeSize.h +++ b/llvm/include/llvm/Support/TypeSize.h @@ -25,148 +25,58 @@ template struct DenseMapInfo; -class ElementCount { -private: - unsigned Min; // Minimum number of vector elements. - bool Scalable; // If true, NumElements is a multiple of 'Min' determined - // at runtime rather than compile time. - - /// Prevent code from using initializer-list contructors like - /// ElementCount EC = {, }. The static `get*` - /// methods below are preferred, as users should always make a - /// conscious choice on the type of `ElementCount` they are - /// requesting. - ElementCount(unsigned Min, bool Scalable) : Min(Min), Scalable(Scalable) {} +// TODO: This class will be redesigned in a later patch that introduces full +// polynomial behaviour, i.e. the ability to have composites made up of both +// fixed and scalable sizes. +template class PolySize { +protected: + T MinVal; // The minimum value that it could be. + bool IsScalable; // If true, the total value is determined by multiplying + // 'MinVal' by a runtime determinded quantity, 'vscale'. + + constexpr PolySize(T MinVal, bool IsScalable) + : MinVal(MinVal), IsScalable(IsScalable) {} public: - ElementCount() = default; - ElementCount operator*(unsigned RHS) { - return { Min * RHS, Scalable }; + static constexpr PolySize getFixed(T MinVal) { return {MinVal, false}; } + static constexpr PolySize getScalable(T MinVal) { return {MinVal, true}; } + static constexpr PolySize get(T MinVal, bool IsScalable) { + return {MinVal, IsScalable}; } - friend ElementCount operator-(const ElementCount &LHS, - const ElementCount &RHS) { - assert(LHS.Scalable == RHS.Scalable && - "Arithmetic using mixed scalable and fixed types"); - return {LHS.Min - RHS.Min, LHS.Scalable}; - } - - bool operator==(const ElementCount& RHS) const { - return Min == RHS.Min && Scalable == RHS.Scalable; - } - bool operator!=(const ElementCount& RHS) const { - return !(*this == RHS); - } - bool operator==(unsigned RHS) const { return Min == RHS && !Scalable; } - bool operator!=(unsigned RHS) const { return !(*this == RHS); } - - ElementCount &operator*=(unsigned RHS) { - Min *= RHS; - return *this; - } - - /// We do not provide the '/' operator here because division for polynomial - /// types does not work in the same way as for normal integer types. We can - /// only divide the minimum value (or coefficient) by RHS, which is not the - /// same as - /// (Min * Vscale) / RHS - /// The caller is recommended to use this function in combination with - /// isKnownMultipleOf(RHS), which lets the caller know if it's possible to - /// perform a lossless divide by RHS. - ElementCount divideCoefficientBy(unsigned RHS) const { - return ElementCount(Min / RHS, Scalable); - } + static constexpr PolySize getNull() { return {0, false}; } - ElementCount NextPowerOf2() const { - return {(unsigned)llvm::NextPowerOf2(Min), Scalable}; - } - - /// This function tells the caller whether the element count is known at - /// compile time to be a multiple of the scalar value RHS. - bool isKnownMultipleOf(unsigned RHS) const { - return Min % RHS == 0; - } - - static ElementCount getFixed(unsigned Min) { return {Min, false}; } - static ElementCount getScalable(unsigned Min) { return {Min, true}; } - static ElementCount get(unsigned Min, bool Scalable) { - return {Min, Scalable}; - } - - /// Printing function. - void print(raw_ostream &OS) const { - if (Scalable) - OS << "vscale x "; - OS << Min; - } /// Counting predicates. /// - /// Notice that Min = 1 and Scalable = true is considered more than - /// one element. - /// ///@{ No elements.. - bool isZero() const { return Min == 0; } + bool isZero() const { return MinVal == 0; } /// At least one element. - bool isNonZero() const { return Min != 0; } + bool isNonZero() const { return !isZero(); } /// A return value of true indicates we know at compile time that the number /// of elements (vscale * Min) is definitely even. However, returning false /// does not guarantee that the total number of elements is odd. - bool isKnownEven() const { return (Min & 0x1) == 0; } - /// Exactly one element. - bool isScalar() const { return !Scalable && Min == 1; } - /// One or more elements. - bool isVector() const { return (Scalable && Min != 0) || Min > 1; } + bool isKnownEven() const { return (MinVal & 0x1) == 0; } ///@} - unsigned getKnownMinValue() const { return Min; } + T getKnownMinValue() const { return MinVal; } // Return the minimum value with the assumption that the count is exact. // Use in places where a scalable count doesn't make sense (e.g. non-vector // types, or vectors in backends which don't support scalable vectors). - unsigned getFixedValue() const { - assert(!Scalable && + T getFixedValue() const { + assert(!IsScalable && "Request for a fixed element count on a scalable object"); - return Min; - } - - bool isScalable() const { return Scalable; } -}; - -/// Stream operator function for `ElementCount`. -inline raw_ostream &operator<<(raw_ostream &OS, const ElementCount &EC) { - EC.print(OS); - return OS; -} - -// This class is used to represent the size of types. If the type is of fixed -// size, it will represent the exact size. If the type is a scalable vector, -// it will represent the known minimum size. -class TypeSize { - uint64_t MinSize; // The known minimum size. - bool IsScalable; // If true, then the runtime size is an integer multiple - // of MinSize. - -public: - constexpr TypeSize(uint64_t MinSize, bool Scalable) - : MinSize(MinSize), IsScalable(Scalable) {} - - static constexpr TypeSize Fixed(uint64_t Size) { - return TypeSize(Size, /*Scalable=*/false); + return MinVal; } - static constexpr TypeSize Scalable(uint64_t MinSize) { - return TypeSize(MinSize, /*Scalable=*/true); - } + bool isScalable() const { return IsScalable; } - // Scalable vector types with the same minimum size as a fixed size type are - // not guaranteed to be the same size at runtime, so they are never - // considered to be equal. - bool operator==(const TypeSize &RHS) const { - return MinSize == RHS.MinSize && IsScalable == RHS.IsScalable; + bool operator==(const PolySize &RHS) const { + return MinVal == RHS.MinVal && IsScalable == RHS.IsScalable; } - bool operator!=(const TypeSize &RHS) const { return !(*this == RHS); } + bool operator!=(const PolySize &RHS) const { return !(*this == RHS); } // For some cases, size ordering between scalable and fixed size types cannot // be determined at compile time, so such comparisons aren't allowed. @@ -178,43 +88,129 @@ // All the functions below make use of the fact vscale is always >= 1, which // means that is guaranteed to be >= <4 x i32>, etc. - static bool isKnownLT(const TypeSize &LHS, const TypeSize &RHS) { + static bool isKnownLT(const PolySize &LHS, const PolySize &RHS) { if (!LHS.IsScalable || RHS.IsScalable) - return LHS.MinSize < RHS.MinSize; + return LHS.MinVal < RHS.MinVal; // LHS.IsScalable = true, RHS.IsScalable = false return false; } - static bool isKnownGT(const TypeSize &LHS, const TypeSize &RHS) { + static bool isKnownGT(const PolySize &LHS, const PolySize &RHS) { if (LHS.IsScalable || !RHS.IsScalable) - return LHS.MinSize > RHS.MinSize; + return LHS.MinVal > RHS.MinVal; // LHS.IsScalable = false, RHS.IsScalable = true return false; } - static bool isKnownLE(const TypeSize &LHS, const TypeSize &RHS) { + static bool isKnownLE(const PolySize &LHS, const PolySize &RHS) { if (!LHS.IsScalable || RHS.IsScalable) - return LHS.MinSize <= RHS.MinSize; + return LHS.MinVal <= RHS.MinVal; // LHS.IsScalable = true, RHS.IsScalable = false return false; } - static bool isKnownGE(const TypeSize &LHS, const TypeSize &RHS) { + static bool isKnownGE(const PolySize &LHS, const PolySize &RHS) { if (LHS.IsScalable || !RHS.IsScalable) - return LHS.MinSize >= RHS.MinSize; + return LHS.MinVal >= RHS.MinVal; // LHS.IsScalable = false, RHS.IsScalable = true return false; } + PolySize operator*(T RHS) { return {MinVal * RHS, IsScalable}; } + + PolySize &operator*=(T RHS) { + MinVal *= RHS; + return *this; + } + + friend PolySize operator-(const PolySize &LHS, const PolySize &RHS) { + assert(LHS.IsScalable == RHS.IsScalable && + "Arithmetic using mixed scalable and fixed types"); + return {LHS.MinVal - RHS.MinVal, LHS.IsScalable}; + } + + /// This function tells the caller whether the element count is known at + /// compile time to be a multiple of the scalar value RHS. + bool isKnownMultipleOf(T RHS) const { return MinVal % RHS == 0; } + + /// We do not provide the '/' operator here because division for polynomial + /// types does not work in the same way as for normal integer types. We can + /// only divide the minimum value (or coefficient) by RHS, which is not the + /// same as + /// (Min * Vscale) / RHS + /// The caller is recommended to use this function in combination with + /// isKnownMultipleOf(RHS), which lets the caller know if it's possible to + /// perform a lossless divide by RHS. + PolySize divideCoefficientBy(T RHS) const { + return PolySize(MinVal / RHS, IsScalable); + } + + PolySize coefficientNextPowerOf2() const { + return PolySize(static_cast(llvm::NextPowerOf2(MinVal)), IsScalable); + } + + /// Printing function. + void print(raw_ostream &OS) const { + if (IsScalable) + OS << "vscale x "; + OS << MinVal; + } +}; + +/// Stream operator function for `PolySize`. +template +inline raw_ostream &operator<<(raw_ostream &OS, const PolySize &PS) { + PS.print(OS); + return OS; +} + +class ElementCount : public PolySize { +public: + + constexpr ElementCount(PolySize V) : PolySize(V) {} + + /// Counting predicates. + /// + /// Notice that MinVal = 1 and IsScalable = true is considered more than + /// one element. + /// + ///@{ No elements.. + /// Exactly one element. + bool isScalar() const { return !IsScalable && MinVal == 1; } + /// One or more elements. + bool isVector() const { return (IsScalable && MinVal != 0) || MinVal > 1; } + ///@} +}; + +// This class is used to represent the size of types. If the type is of fixed +// size, it will represent the exact size. If the type is a scalable vector, +// it will represent the known minimum size. +class TypeSize : public PolySize { +public: + constexpr TypeSize(PolySize V) : PolySize(V) {} + + constexpr TypeSize(uint64_t MinVal, bool IsScalable) + : PolySize(MinVal, IsScalable) {} + + static constexpr TypeSize Fixed(uint64_t MinVal) { + return TypeSize(MinVal, false); + } + static constexpr TypeSize Scalable(uint64_t MinVal) { + return TypeSize(MinVal, true); + } + + uint64_t getFixedSize() const { return getFixedValue(); } + uint64_t getKnownMinSize() const { return getKnownMinValue(); } + friend bool operator<(const TypeSize &LHS, const TypeSize &RHS) { assert(LHS.IsScalable == RHS.IsScalable && "Ordering comparison of scalable and fixed types"); - return LHS.MinSize < RHS.MinSize; + return LHS.MinVal < RHS.MinVal; } friend bool operator>(const TypeSize &LHS, const TypeSize &RHS) { @@ -229,83 +225,26 @@ return !(LHS < RHS); } - // Convenience operators to obtain relative sizes independently of - // the scalable flag. - TypeSize operator*(unsigned RHS) const { - return { MinSize * RHS, IsScalable }; - } - - friend TypeSize operator*(const unsigned LHS, const TypeSize &RHS) { - return { LHS * RHS.MinSize, RHS.IsScalable }; - } - - /// We do not provide the '/' operator here because division for polynomial - /// types does not work in the same way as for normal integer types. We can - /// only divide the minimum value (or coefficient) by RHS, which is not the - /// same as - /// (MinSize * Vscale) / RHS - /// The caller is recommended to use this function in combination with - /// isKnownMultipleOf(RHS), which lets the caller know if it's possible to - /// perform a lossless divide by RHS. - TypeSize divideCoefficientBy(uint64_t RHS) const { - return {MinSize / RHS, IsScalable}; - } - TypeSize &operator-=(TypeSize RHS) { assert(IsScalable == RHS.IsScalable && "Subtraction using mixed scalable and fixed types"); - MinSize -= RHS.MinSize; + MinVal -= RHS.MinVal; return *this; } TypeSize &operator+=(TypeSize RHS) { assert(IsScalable == RHS.IsScalable && "Addition using mixed scalable and fixed types"); - MinSize += RHS.MinSize; + MinVal += RHS.MinVal; return *this; } friend TypeSize operator-(const TypeSize &LHS, const TypeSize &RHS) { assert(LHS.IsScalable == RHS.IsScalable && "Arithmetic using mixed scalable and fixed types"); - return {LHS.MinSize - RHS.MinSize, LHS.IsScalable}; - } - - // Return the minimum size with the assumption that the size is exact. - // Use in places where a scalable size doesn't make sense (e.g. non-vector - // types, or vectors in backends which don't support scalable vectors). - uint64_t getFixedSize() const { - assert(!IsScalable && "Request for a fixed size on a scalable object"); - return MinSize; - } - - // Return the known minimum size. Use in places where the scalable property - // doesn't matter (e.g. determining alignment) or in conjunction with the - // isScalable method below. - uint64_t getKnownMinSize() const { - return MinSize; - } - - // Return whether or not the size is scalable. - bool isScalable() const { - return IsScalable; + return {LHS.MinVal - RHS.MinVal, LHS.IsScalable}; } - // Returns true if the number of bits is a multiple of an 8-bit byte. - bool isByteSized() const { - return (MinSize & 7) == 0; - } - - // Returns true if the type size is non-zero. - bool isNonZero() const { return MinSize != 0; } - - // Returns true if the type size is zero. - bool isZero() const { return MinSize == 0; } - - /// This function tells the caller whether the type size is known at - /// compile time to be a multiple of the scalar value RHS. - bool isKnownMultipleOf(uint64_t RHS) const { return MinSize % RHS == 0; } - // Casts to a uint64_t if this is a fixed-width size. // // This interface is deprecated and will be removed in a future version @@ -317,53 +256,51 @@ // To determine how to upgrade the code: // // if () - // use getKnownMinSize() + // use getKnownMinValue() // else if () { // if - // update the algorithm and use getKnownMinSize() + // update the algorithm and use getKnownMinValue() // else - // bail out early for scalable vectors and use getFixedSize() + // bail out early for scalable vectors and use getFixedValue() // } operator uint64_t() const { #ifdef STRICT_FIXED_SIZE_VECTORS - return getFixedSize(); + return getFixedValue(); #else if (isScalable()) WithColor::warning() << "Compiler has made implicit assumption that " "TypeSize is not scalable. This may or may not " "lead to broken code.\n"; - return getKnownMinSize(); + return getKnownMinValue(); #endif } + // Convenience operators to obtain relative sizes independently of + // the scalable flag. + TypeSize operator*(unsigned RHS) const { return {MinVal * RHS, IsScalable}; } + + friend TypeSize operator*(const unsigned LHS, const TypeSize &RHS) { + return {LHS * RHS.MinVal, RHS.IsScalable}; + } + // Additional convenience operators needed to avoid ambiguous parses. // TODO: Make uint64_t the default operator? - TypeSize operator*(uint64_t RHS) const { - return { MinSize * RHS, IsScalable }; - } + TypeSize operator*(uint64_t RHS) const { return {MinVal * RHS, IsScalable}; } - TypeSize operator*(int RHS) const { - return { MinSize * RHS, IsScalable }; - } + TypeSize operator*(int RHS) const { return {MinVal * RHS, IsScalable}; } - TypeSize operator*(int64_t RHS) const { - return { MinSize * RHS, IsScalable }; - } + TypeSize operator*(int64_t RHS) const { return {MinVal * RHS, IsScalable}; } friend TypeSize operator*(const uint64_t LHS, const TypeSize &RHS) { - return { LHS * RHS.MinSize, RHS.IsScalable }; + return {LHS * RHS.MinVal, RHS.IsScalable}; } friend TypeSize operator*(const int LHS, const TypeSize &RHS) { - return { LHS * RHS.MinSize, RHS.IsScalable }; + return {LHS * RHS.MinVal, RHS.IsScalable}; } friend TypeSize operator*(const int64_t LHS, const TypeSize &RHS) { - return { LHS * RHS.MinSize, RHS.IsScalable }; - } - - TypeSize NextPowerOf2() const { - return TypeSize(llvm::NextPowerOf2(MinSize), IsScalable); + return {LHS * RHS.MinVal, RHS.IsScalable}; } }; @@ -374,7 +311,7 @@ /// Similar to the alignTo functions in MathExtras.h inline TypeSize alignTo(TypeSize Size, uint64_t Align) { assert(Align != 0u && "Align must be non-zero"); - return {(Size.getKnownMinSize() + Align - 1) / Align * Align, + return {(Size.getKnownMinValue() + Align - 1) / Align * Align, Size.isScalable()}; } diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -19573,7 +19573,7 @@ V.getOperand(0), NewIndex); return DAG.getBitcast(NVT, NewExtract); } - if (NewExtEC == 1 && + if (NewExtEC.isScalar() && TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, ScalarVT)) { SDValue NewIndex = DAG.getVectorIdxConstant(IndexValScaled, DL); SDValue NewExtract = diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp --- a/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -862,7 +862,7 @@ EVT EltVT = VT.getVectorElementType(); // Vectors with only one element are always scalarized. - if (NumElts == 1) + if (NumElts.isScalar()) return LegalizeKind(TypeScalarizeVector, EltVT); if (VT.getVectorElementCount() == ElementCount::getScalable(1)) @@ -875,7 +875,7 @@ // Vectors with a number of elements that is not a power of two are always // widened, for example <3 x i8> -> <4 x i8>. if (!VT.isPow2VectorType()) { - NumElts = NumElts.NextPowerOf2(); + NumElts = NumElts.coefficientNextPowerOf2(); EVT NVT = EVT::getVectorVT(Context, EltVT, NumElts); return LegalizeKind(TypeWidenVector, NVT); } @@ -924,7 +924,7 @@ // If there is no wider legal type, split the vector. while (true) { // Round up to the next power of 2. - NumElts = NumElts.NextPowerOf2(); + NumElts = NumElts.coefficientNextPowerOf2(); // If there is no simple vector type with this many elements then there // cannot be a larger legal vector type. Note that this assumes that @@ -1499,7 +1499,7 @@ TypeSize NewVTSize = NewVT.getSizeInBits(); // Convert sizes such as i33 to i64. if (!isPowerOf2_32(NewVTSize.getKnownMinSize())) - NewVTSize = NewVTSize.NextPowerOf2(); + NewVTSize = NewVTSize.coefficientNextPowerOf2(); return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits()); } diff --git a/llvm/lib/Support/LowLevelType.cpp b/llvm/lib/Support/LowLevelType.cpp --- a/llvm/lib/Support/LowLevelType.cpp +++ b/llvm/lib/Support/LowLevelType.cpp @@ -23,7 +23,7 @@ } else if (VT.isValid()) { // Aggregates are no different from real scalars as far as GlobalISel is // concerned. - assert(VT.getSizeInBits() != 0 && "invalid zero-sized type"); + assert(VT.getSizeInBits().isNonZero() && "invalid zero-sized type"); init(/*IsPointer=*/false, /*IsVector=*/false, /*NumElements=*/0, VT.getSizeInBits(), /*AddressSpace=*/0); } else { diff --git a/llvm/lib/Target/X86/X86InterleavedAccess.cpp b/llvm/lib/Target/X86/X86InterleavedAccess.cpp --- a/llvm/lib/Target/X86/X86InterleavedAccess.cpp +++ b/llvm/lib/Target/X86/X86InterleavedAccess.cpp @@ -211,7 +211,7 @@ VecBasePtr = Builder.CreateBitCast(LI->getPointerOperand(), VecBasePtrTy); } // Generate N loads of T type. - assert(VecBaseTy->getPrimitiveSizeInBits().isByteSized() && + assert(VecBaseTy->getPrimitiveSizeInBits().isKnownMultipleOf(8) && "VecBaseTy's size must be a multiple of 8"); const Align FirstAlignment = LI->getAlign(); const Align SubsequentAlignment = commonAlignment( diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -353,7 +353,9 @@ // with a vector. if (VF.isVector()) { auto *VectorTy = VectorType::get(Ty, VF); - return VF * DL.getTypeAllocSize(Ty) != DL.getTypeStoreSize(VectorTy); + return TypeSize::get(VF.getKnownMinValue() * + DL.getTypeAllocSize(Ty).getFixedValue(), + VF.isScalable()) != DL.getTypeStoreSize(VectorTy); } // If the vectorization factor is one, we just check if an array of type Ty @@ -2166,7 +2168,7 @@ // If we aren't vectorizing, we can just copy the scalar map values over to // the vector map. - if (VF == 1) { + if (VF.isScalar()) { VectorLoopValueMap.setVectorValue(V, Part, ScalarValue); return ScalarValue; } @@ -2242,7 +2244,7 @@ // extractelement instruction. auto *U = getOrCreateVectorValue(V, Instance.Part); if (!U->getType()->isVectorTy()) { - assert(VF == 1 && "Value not scalarized has non-vector type"); + assert(VF.isScalar() && "Value not scalarized has non-vector type"); return U; } @@ -3933,7 +3935,7 @@ if (RK == RecurrenceDescriptor::RK_IntegerMinMax || RK == RecurrenceDescriptor::RK_FloatMinMax) { // MinMax reduction have the start value as their identify. - if (VF == 1 || IsInLoopReductionPhi) { + if (VF.isScalar() || IsInLoopReductionPhi) { VectorStart = Identity = ReductionStartValue; } else { VectorStart = Identity = @@ -3943,7 +3945,7 @@ // Handle other reduction kinds: Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity( RK, MinMaxKind, VecTy->getScalarType()); - if (VF == 1 || IsInLoopReductionPhi) { + if (VF.isScalar() || IsInLoopReductionPhi) { Identity = Iden; // This vector is the Identity vector where the first element is the // incoming scalar reduction. @@ -4343,7 +4345,7 @@ ? Builder.CreateInBoundsGEP(GEP->getSourceElementType(), Ptr, Indices) : Builder.CreateGEP(GEP->getSourceElementType(), Ptr, Indices); - assert((VF == 1 || NewGEP->getType()->isVectorTy()) && + assert((VF.isScalar() || NewGEP->getType()->isVectorTy()) && "NewGEP is not a pointer vector"); VectorLoopValueMap.setVectorValue(GEP, Part, NewGEP); addMetadata(NewGEP, GEP); @@ -8413,7 +8415,7 @@ return false; } - if (VF.Width == 1) { + if (VF.Width.isScalar()) { LLVM_DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n"); VecDiagMsg = std::make_pair( "VectorizationNotBeneficial", diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -536,7 +536,7 @@ "trip.count.minus.1"); auto VF = State->VF; Value *VTCMO = - VF == 1 ? TCMO : Builder.CreateVectorSplat(VF, TCMO, "broadcast"); + VF.isScalar() ? TCMO : Builder.CreateVectorSplat(VF, TCMO, "broadcast"); for (unsigned Part = 0, UF = State->UF; Part < UF; ++Part) State->set(BackedgeTakenCount, VTCMO, Part); } @@ -930,7 +930,8 @@ ConstantInt::get(STy, Part * VF.getKnownMinValue() + Lane)); // If VF == 1, there is only one iteration in the loop above, thus the // element pushed back into Indices is ConstantInt::get(STy, Part) - Constant *VStep = VF == 1 ? Indices.back() : ConstantVector::get(Indices); + Constant *VStep = + VF.isScalar() ? Indices.back() : ConstantVector::get(Indices); // Add the consecutive indices to the vector value. Value *CanonicalVectorIV = Builder.CreateAdd(VStart, VStep, "vec.iv"); State.set(getVPValue(), CanonicalVectorIV, Part);