Index: include/llvm/Support/LowLevelTypeImpl.h =================================================================== --- include/llvm/Support/LowLevelTypeImpl.h +++ include/llvm/Support/LowLevelTypeImpl.h @@ -39,100 +39,87 @@ class LLT { public: - enum TypeKind : uint16_t { - Invalid, - Scalar, - Pointer, - Vector, - }; - /// Get a low-level scalar or aggregate "bag of bits". static LLT scalar(unsigned SizeInBits) { assert(SizeInBits > 0 && "invalid scalar size"); - return LLT{Scalar, 1, SizeInBits}; + return LLT{false, false, 0, SizeInBits, 0}; } /// Get a low-level pointer in the given address space (defaulting to 0). static LLT pointer(uint16_t AddressSpace, unsigned SizeInBits) { - return LLT{Pointer, AddressSpace, SizeInBits}; + assert(SizeInBits > 0 && "invalid pointer size"); + return LLT{true, false, 0, SizeInBits, AddressSpace}; } /// Get a low-level vector of some number of elements and element width. /// \p NumElements must be at least 2. static LLT vector(uint16_t NumElements, unsigned ScalarSizeInBits) { assert(NumElements > 1 && "invalid number of vector elements"); - return LLT{Vector, NumElements, ScalarSizeInBits}; + assert(ScalarSizeInBits > 0 && "invalid vector element size"); + return LLT{false, true, NumElements, ScalarSizeInBits, 0}; } /// Get a low-level vector of some number of elements and element type. static LLT vector(uint16_t NumElements, LLT ScalarTy) { assert(NumElements > 1 && "invalid number of vector elements"); - assert(ScalarTy.isScalar() && "invalid vector element type"); - return LLT{Vector, NumElements, ScalarTy.getSizeInBits()}; + assert(!ScalarTy.isVector() && "invalid vector element type"); + return LLT{ScalarTy.isPointer(), true, NumElements, + ScalarTy.getSizeInBits(), + ScalarTy.isPointer() ? ScalarTy.getAddressSpace() : 0}; } - explicit LLT(TypeKind Kind, uint16_t NumElements, unsigned SizeInBits) - : SizeInBits(SizeInBits), ElementsOrAddrSpace(NumElements), Kind(Kind) { - assert((Kind != Vector || ElementsOrAddrSpace > 1) && - "invalid number of vector elements"); + explicit LLT(bool isPointer, bool isVector, uint16_t NumElements, + unsigned SizeInBits, unsigned AddressSpace) { + init(isPointer, isVector, NumElements, SizeInBits, AddressSpace); } - - explicit LLT() : SizeInBits(0), ElementsOrAddrSpace(0), Kind(Invalid) {} + explicit LLT() : IsPointer(false), IsVector(false), RawData(0) {} explicit LLT(MVT VT); - bool isValid() const { return Kind != Invalid; } - - bool isScalar() const { return Kind == Scalar; } + bool isValid() const { return RawData != 0; } - bool isPointer() const { return Kind == Pointer; } + bool isScalar() const { return isValid() && !IsPointer && !IsVector; } - bool isVector() const { return Kind == Vector; } + bool isPointer() const { return isValid() && IsPointer && !IsVector; } - /// Returns the number of elements in a vector LLT. Must only be called on - /// vector types. - uint16_t getNumElements() const { - assert(isVector() && "cannot get number of elements on scalar/aggregate"); - return ElementsOrAddrSpace; - } + bool isVector() const { return isValid() && IsVector; } /// Returns the total size of the type. Must only be called on sized types. unsigned getSizeInBits() const { if (isPointer() || isScalar()) - return SizeInBits; - return SizeInBits * ElementsOrAddrSpace; + return getScalarSizeInBits(); + return getScalarSizeInBits() * getNumElements(); } - unsigned getScalarSizeInBits() const { - return SizeInBits; - } - - unsigned getAddressSpace() const { - assert(isPointer() && "cannot get address space of non-pointer type"); - return ElementsOrAddrSpace; - } /// Returns the vector's element type. Only valid for vector types. LLT getElementType() const { assert(isVector() && "cannot get element type of scalar/aggregate"); - return scalar(SizeInBits); + if (IsPointer) + return pointer(getAddressSpace(), getScalarSizeInBits()); + else + return scalar(getScalarSizeInBits()); } /// Get a low-level type with half the size of the original, by halving the /// size of the scalar type involved. For example `s32` will become `s16`, /// `<2 x s32>` will become `<2 x s16>`. LLT halfScalarSize() const { - assert(!isPointer() && getScalarSizeInBits() > 1 && + assert(!IsPointer && getScalarSizeInBits() > 1 && getScalarSizeInBits() % 2 == 0 && "cannot half size of this type"); - return LLT{Kind, ElementsOrAddrSpace, SizeInBits / 2}; + return LLT{false, IsVector ? true : false, + IsVector ? getNumElements() : (uint16_t)0, + getScalarSizeInBits() / 2, 0}; } /// Get a low-level type with twice the size of the original, by doubling the /// size of the scalar type involved. For example `s32` will become `s64`, /// `<2 x s32>` will become `<2 x s64>`. LLT doubleScalarSize() const { - assert(!isPointer() && "cannot change size of this type"); - return LLT{Kind, ElementsOrAddrSpace, SizeInBits * 2}; + assert(!IsPointer && "cannot change size of this type"); + return LLT{false, IsVector ? true : false, + IsVector ? getNumElements() : (uint16_t)0, + getScalarSizeInBits() * 2, 0}; } /// Get a low-level type with half the size of the original, by halving the @@ -140,13 +127,12 @@ /// a vector type with an even number of elements. For example `<4 x s32>` /// will become `<2 x s32>`, `<2 x s32>` will become `s32`. LLT halfElements() const { - assert(isVector() && ElementsOrAddrSpace % 2 == 0 && - "cannot half odd vector"); - if (ElementsOrAddrSpace == 2) - return scalar(SizeInBits); + assert(isVector() && getNumElements() % 2 == 0 && "cannot half odd vector"); + if (getNumElements() == 2) + return scalar(getScalarSizeInBits()); - return LLT{Vector, static_cast(ElementsOrAddrSpace / 2), - SizeInBits}; + return LLT{false, true, (uint16_t)(getNumElements() / 2), + getScalarSizeInBits(), 0}; } /// Get a low-level type with twice the size of the original, by doubling the @@ -154,25 +140,105 @@ /// a vector type. For example `<2 x s32>` will become `<4 x s32>`. Doubling /// the number of elements in sN produces <2 x sN>. LLT doubleElements() const { - assert(!isPointer() && "cannot double elements in pointer"); - return LLT{Vector, static_cast(ElementsOrAddrSpace * 2), - SizeInBits}; + return LLT{IsPointer ? true : false, true, (uint16_t)(getNumElements() * 2), + getScalarSizeInBits(), IsPointer ? getAddressSpace() : 0}; } void print(raw_ostream &OS) const; bool operator==(const LLT &RHS) const { - return Kind == RHS.Kind && SizeInBits == RHS.SizeInBits && - ElementsOrAddrSpace == RHS.ElementsOrAddrSpace; + return IsPointer == RHS.IsPointer && IsVector == RHS.IsVector && + RHS.RawData == RawData; } bool operator!=(const LLT &RHS) const { return !(*this == RHS); } friend struct DenseMapInfo; private: - unsigned SizeInBits; - uint16_t ElementsOrAddrSpace; - TypeKind Kind; + /// LLT is packed into 64 bits as follows: + /// isPointer : 1 + /// isVector : 1 + /// with 62 bits remaining for Kind-specific data: + /// Non-pointer scalar (isPointer == 0 && isVector == 0): + /// SizeInBits: 32; + /// Pointer (isPointer == 1 && isVector == 0): + /// SizeInBits: 32; + /// AddressSpace: 23; + /// Vector-of-non-pointer (isPointer == 0 && isVector == 1): + /// NumElements: 16; + /// sizeOfElement: 32; + /// Vector-of-pointer (isPointer == 1 && isVector == 1): + /// NumElements: 16; + /// SizeOfElement: 16; + /// AddressSpace: 23; + /// + /// Invalid gets encoded as RawData == 0, as that is an invalid encoding, + /// since for valid encodings, SizeInBits/SizeOfElement must be larger than 0. + uint64_t IsPointer : 1; + uint64_t IsVector : 1; + uint64_t RawData : 62; + + static uint64_t maskAndShift(uint64_t val, uint64_t mask, uint8_t shift) { + return (val & mask) << shift; + } + void init(bool isPointer, bool isVector, uint16_t NumElements, + unsigned SizeInBits, unsigned AddressSpace) { + IsPointer = isPointer; + IsVector = isVector; + assert((!IsVector || NumElements > 1) && + "invalid number of vector elements"); + assert(AddressSpace <= 0x7FFFFF && "AddressSpace must fit in 23 bits"); + assert(NumElements <= 0xFFFF && "NumElements must fit in 16 bits"); + if (!IsVector) { + if (!IsPointer) + RawData = maskAndShift(SizeInBits, 0xFFFFFFFF, 0); + else + RawData = maskAndShift(SizeInBits, 0xFFFFFFFF, 0) | + maskAndShift(AddressSpace, 0x7FFFFF, 32); + } else { + assert(SizeInBits <= 0xFFFF && + "SizeInBits must fit in 16 bits for vectors"); + if (!IsPointer) + RawData = maskAndShift(NumElements, 0xFFFF, 0) | + maskAndShift(SizeInBits, 0xFFFFFFFF, 16); + else + RawData = maskAndShift(NumElements, 0xFFFF, 0) | + maskAndShift(SizeInBits, 0xFFFF, 16) | + maskAndShift(AddressSpace, 0x7FFFFF, 32); + } + } + +public: + unsigned getScalarSizeInBits() const { + assert(RawData != 0); + if (!IsVector) { + if (!IsPointer) + return 0xFFFFFFFF & (RawData >> 0); + else + return 0xFFFFFFFF & (RawData >> 0); + } else { + if (!IsPointer) + return 0xFFFFFFFF & (RawData >> 16); + else + return 0xFFFF & (RawData >> 16); + } + } + + unsigned getAddressSpace() const { + assert(RawData != 0); + assert(IsPointer && "cannot get address space of non-pointer type"); + if (!IsVector) + return 0x7FFFFF & (RawData >> 32); + else + return 0x7FFFFF & (RawData >> (16 + 16)); + } + + /// Returns the number of elements in a vector LLT. Must only be called on + /// vector types. + uint16_t getNumElements() const { + assert(IsVector && "cannot get number of elements on scalar/aggregate"); + return 0xFFFF & (RawData >> 0); + } }; inline raw_ostream& operator<<(raw_ostream &OS, const LLT &Ty) { @@ -182,14 +248,18 @@ template<> struct DenseMapInfo { static inline LLT getEmptyKey() { - return LLT{LLT::Invalid, 0, -1u}; + LLT Invalid; + Invalid.IsPointer = true; + return Invalid; } static inline LLT getTombstoneKey() { - return LLT{LLT::Invalid, 0, -2u}; + LLT Invalid; + Invalid.IsVector = true; + return Invalid; } static inline unsigned getHashValue(const LLT &Ty) { - uint64_t Val = ((uint64_t)Ty.SizeInBits << 32) | - ((uint64_t)Ty.ElementsOrAddrSpace << 16) | (uint64_t)Ty.Kind; + uint64_t Val = ((uint64_t)Ty.RawData) << 2 | ((uint64_t)Ty.IsPointer) << 1 | + ((uint64_t)Ty.IsVector); return DenseMapInfo::getHashValue(Val); } static bool isEqual(const LLT &LHS, const LLT &RHS) { Index: lib/CodeGen/GlobalISel/MachineIRBuilder.cpp =================================================================== --- lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -592,7 +592,7 @@ LLT EltTy = MRI->getType(Elt); LLT IdxTy = MRI->getType(Idx); assert(ResTy.isVector() && ValTy.isVector() && "invalid operand type"); - assert(EltTy.isScalar() && IdxTy.isScalar() && "invalid operand type"); + assert(IdxTy.isScalar() && "invalid operand type"); assert(ResTy.getNumElements() == ValTy.getNumElements() && "type mismatch"); assert(ResTy.getElementType() == EltTy && "type mismatch"); #endif @@ -612,7 +612,8 @@ LLT ValTy = MRI->getType(Val); LLT IdxTy = MRI->getType(Idx); assert(ValTy.isVector() && "invalid operand type"); - assert(ResTy.isScalar() && IdxTy.isScalar() && "invalid operand type"); + assert((ResTy.isScalar() || ResTy.isPointer()) && "invalid operand type"); + assert(IdxTy.isScalar() && "invalid operand type"); assert(ValTy.getElementType() == ResTy && "type mismatch"); #endif Index: lib/CodeGen/LowLevelType.cpp =================================================================== --- lib/CodeGen/LowLevelType.cpp +++ lib/CodeGen/LowLevelType.cpp @@ -21,10 +21,10 @@ LLT llvm::getLLTForType(Type &Ty, const DataLayout &DL) { if (auto VTy = dyn_cast(&Ty)) { auto NumElements = VTy->getNumElements(); - auto ScalarSizeInBits = VTy->getElementType()->getPrimitiveSizeInBits(); + LLT ScalarTy = getLLTForType(*VTy->getElementType(), DL); if (NumElements == 1) - return LLT::scalar(ScalarSizeInBits); - return LLT::vector(NumElements, ScalarSizeInBits); + return ScalarTy; + return LLT::vector(NumElements, ScalarTy); } else if (auto PTy = dyn_cast(&Ty)) { return LLT::pointer(PTy->getAddressSpace(), DL.getTypeSizeInBits(&Ty)); } else if (Ty.isSized()) { Index: lib/Support/LowLevelType.cpp =================================================================== --- lib/Support/LowLevelType.cpp +++ lib/Support/LowLevelType.cpp @@ -18,25 +18,23 @@ LLT::LLT(MVT VT) { if (VT.isVector()) { - SizeInBits = VT.getVectorElementType().getSizeInBits(); - ElementsOrAddrSpace = VT.getVectorNumElements(); - Kind = ElementsOrAddrSpace == 1 ? Scalar : Vector; + init(false, VT.getVectorNumElements() > 1, VT.getVectorNumElements(), + VT.getVectorElementType().getSizeInBits(), 0); } else if (VT.isValid()) { // Aggregates are no different from real scalars as far as GlobalISel is // concerned. - Kind = Scalar; - SizeInBits = VT.getSizeInBits(); - ElementsOrAddrSpace = 1; - assert(SizeInBits != 0 && "invalid zero-sized type"); + assert(VT.getSizeInBits() != 0 && "invalid zero-sized type"); + init(false, false, 0, VT.getSizeInBits(), 0); } else { - Kind = Invalid; - SizeInBits = ElementsOrAddrSpace = 0; + IsPointer = false; + IsVector = false; + RawData = 0; } } void LLT::print(raw_ostream &OS) const { if (isVector()) - OS << "<" << ElementsOrAddrSpace << " x s" << SizeInBits << ">"; + OS << "<" << getNumElements() << " x " << getElementType() << ">"; else if (isPointer()) OS << "p" << getAddressSpace(); else if (isValid()) { Index: lib/Target/AArch64/AArch64RegisterBankInfo.cpp =================================================================== --- lib/Target/AArch64/AArch64RegisterBankInfo.cpp +++ lib/Target/AArch64/AArch64RegisterBankInfo.cpp @@ -486,7 +486,7 @@ continue; LLT Ty = MRI.getType(MO.getReg()); - OpSize[Idx] = Ty.getSizeInBits(); + OpSize[Idx] = Ty.isValid() ? Ty.getSizeInBits() : 0; // As a top-level guess, vectors go in FPRs, scalars and pointers in GPRs. // For floating-point instructions, scalars go in FPRs. Index: test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll =================================================================== --- test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll +++ test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll @@ -146,3 +146,21 @@ define fp128 @test_quad_dump() { ret fp128 0xL00000000000000004000000000000000 } + +; AArch64 was asserting when using vectors of pointers, which cannot be +; represented in LLT at the moment. +; FALLBACK-WITH-REPORT-ERR: remark: :0:0: unable to legalize instruction: %vreg0(p0) = G_EXTRACT_VECTOR_ELT %vreg1, %vreg2; (in function: vector_of_pointers_extractelement) +; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for vector_of_pointers_extractelement +; FALLBACK-WITH-REPORT-OUT-LABEL: vector_of_pointers_extractelement: +define void @vector_of_pointers_extractelement() { + %dummy = extractelement <2 x i16*> undef, i32 0 + ret void +} + +; FALLBACK-WITH-REPORT-ERR: remark: :0:0: unable to legalize instruction: %vreg0(<2 x p0>) = G_INSERT_VECTOR_ELT %vreg1, %vreg2, %vreg3; (in function: vector_of_pointers_insertelement +; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for vector_of_pointers_insertelement +; FALLBACK-WITH-REPORT-OUT-LABEL: vector_of_pointers_insertelement: +define void @vector_of_pointers_insertelement() { + %dummy = insertelement <2 x i16*> undef, i16* null, i32 0 + ret void +} Index: unittests/CodeGen/LowLevelTypeTest.cpp =================================================================== --- unittests/CodeGen/LowLevelTypeTest.cpp +++ unittests/CodeGen/LowLevelTypeTest.cpp @@ -171,6 +171,7 @@ for (unsigned AS : {0U, 1U, 127U, 0xffffU}) { const LLT Ty = LLT::pointer(AS, DL.getPointerSizeInBits(AS)); + const LLT VTy = LLT::vector(4, Ty); // Test kind. ASSERT_TRUE(Ty.isValid()); @@ -179,16 +180,26 @@ ASSERT_FALSE(Ty.isScalar()); ASSERT_FALSE(Ty.isVector()); + ASSERT_TRUE(VTy.isValid()); + ASSERT_TRUE(VTy.isVector()); + ASSERT_TRUE(VTy.getElementType().isPointer()); + // Test addressspace. EXPECT_EQ(AS, Ty.getAddressSpace()); + EXPECT_EQ(AS, VTy.getElementType().getAddressSpace()); // Test equality operators. EXPECT_TRUE(Ty == Ty); EXPECT_FALSE(Ty != Ty); + EXPECT_TRUE(VTy == VTy); + EXPECT_FALSE(VTy != VTy); // Test Type->LLT conversion. Type *IRTy = PointerType::get(IntegerType::get(C, 8), AS); EXPECT_EQ(Ty, getLLTForType(*IRTy, DL)); + Type *IRVTy = + VectorType::get(PointerType::get(IntegerType::get(C, 8), AS), 4); + EXPECT_EQ(VTy, getLLTForType(*IRVTy, DL)); } }