Index: include/llvm/Support/LowLevelTypeImpl.h =================================================================== --- include/llvm/Support/LowLevelTypeImpl.h +++ include/llvm/Support/LowLevelTypeImpl.h @@ -43,37 +43,39 @@ static LLT scalar(unsigned SizeInBits) { assert(SizeInBits > 0 && "invalid scalar size"); return LLT{/*isPointer=*/false, /*isVector=*/false, /*NumElements=*/0, - SizeInBits, /*AddressSpace=*/0}; + SizeInBits, /*AddressSpace=*/0, /*isScalable=*/false}; } /// Get a low-level pointer in the given address space (defaulting to 0). static LLT pointer(uint16_t AddressSpace, unsigned SizeInBits) { assert(SizeInBits > 0 && "invalid pointer size"); return LLT{/*isPointer=*/true, /*isVector=*/false, /*NumElements=*/0, - SizeInBits, AddressSpace}; + SizeInBits, AddressSpace, /*isScalable=*/false}; } /// Get a low-level vector of some number of elements and element width. /// \p NumElements must be at least 2. - static LLT vector(uint16_t NumElements, unsigned ScalarSizeInBits) { + static LLT vector(uint16_t NumElements, unsigned ScalarSizeInBits, + bool Scalable = false) { assert(NumElements > 1 && "invalid number of vector elements"); assert(ScalarSizeInBits > 0 && "invalid vector element size"); return LLT{/*isPointer=*/false, /*isVector=*/true, NumElements, - ScalarSizeInBits, /*AddressSpace=*/0}; + ScalarSizeInBits, /*AddressSpace=*/0, Scalable}; } /// Get a low-level vector of some number of elements and element type. - static LLT vector(uint16_t NumElements, LLT ScalarTy) { + static LLT vector(uint16_t NumElements, LLT ScalarTy, bool Scalable = false) { assert(NumElements > 1 && "invalid number of vector elements"); assert(!ScalarTy.isVector() && "invalid vector element type"); return LLT{ScalarTy.isPointer(), /*isVector=*/true, NumElements, ScalarTy.getSizeInBits(), - ScalarTy.isPointer() ? ScalarTy.getAddressSpace() : 0}; + ScalarTy.isPointer() ? ScalarTy.getAddressSpace() : 0, + Scalable}; } explicit LLT(bool isPointer, bool isVector, uint16_t NumElements, - unsigned SizeInBits, unsigned AddressSpace) { - init(isPointer, isVector, NumElements, SizeInBits, AddressSpace); + unsigned SizeInBits, unsigned AddressSpace, bool Scalable) { + init(isPointer, isVector, NumElements, SizeInBits, AddressSpace, Scalable); } explicit LLT() : IsPointer(false), IsVector(false), RawData(0) {} @@ -137,6 +139,14 @@ return scalar(getScalarSizeInBits()); } + bool isScalable() const { + assert(isVector() && "cannot get scalable property of non-vector"); + if (IsPointer) + return (getFieldValue(PointerVectorScalableFieldInfo) != 0); + else + return (getFieldValue(VectorScalableFieldInfo) != 0); + } + void print(raw_ostream &OS) const; bool operator==(const LLT &RHS) const { @@ -176,19 +186,26 @@ /// * Vector-of-non-pointer (isPointer == 0 && isVector == 1): /// NumElements: 16; /// SizeOfElement: 32; + /// Scalable: 1; static const constexpr BitFieldInfo VectorElementsFieldInfo{16, 0}; static const constexpr BitFieldInfo VectorSizeFieldInfo{ 32, VectorElementsFieldInfo[0] + VectorElementsFieldInfo[1]}; + static const constexpr BitFieldInfo VectorScalableFieldInfo{ + 1, VectorSizeFieldInfo[0] + VectorSizeFieldInfo[1]}; /// * Vector-of-pointer (isPointer == 1 && isVector == 1): /// NumElements: 16; /// SizeOfElement: 16; /// AddressSpace: 23; + /// Scalable: 1; static const constexpr BitFieldInfo PointerVectorElementsFieldInfo{16, 0}; static const constexpr BitFieldInfo PointerVectorSizeFieldInfo{ 16, PointerVectorElementsFieldInfo[1] + PointerVectorElementsFieldInfo[0]}; static const constexpr BitFieldInfo PointerVectorAddressSpaceFieldInfo{ 23, PointerVectorSizeFieldInfo[1] + PointerVectorSizeFieldInfo[0]}; + static const constexpr BitFieldInfo PointerVectorScalableFieldInfo{ + 1, PointerVectorAddressSpaceFieldInfo[0] + + PointerVectorAddressSpaceFieldInfo[1]}; uint64_t IsPointer : 1; uint64_t IsVector : 1; @@ -210,9 +227,10 @@ } void init(bool IsPointer, bool IsVector, uint16_t NumElements, - unsigned SizeInBits, unsigned AddressSpace) { + unsigned SizeInBits, unsigned AddressSpace, bool IsScalable) { this->IsPointer = IsPointer; this->IsVector = IsVector; + uint64_t Scalable = IsScalable ? 1 : 0; if (!IsVector) { if (!IsPointer) RawData = maskAndShift(SizeInBits, ScalarSizeFieldInfo); @@ -223,12 +241,14 @@ assert(NumElements > 1 && "invalid number of vector elements"); if (!IsPointer) RawData = maskAndShift(NumElements, VectorElementsFieldInfo) | - maskAndShift(SizeInBits, VectorSizeFieldInfo); + maskAndShift(SizeInBits, VectorSizeFieldInfo) | + maskAndShift(Scalable, VectorScalableFieldInfo); else RawData = maskAndShift(NumElements, PointerVectorElementsFieldInfo) | maskAndShift(SizeInBits, PointerVectorSizeFieldInfo) | - maskAndShift(AddressSpace, PointerVectorAddressSpaceFieldInfo); + maskAndShift(AddressSpace, PointerVectorAddressSpaceFieldInfo) | + maskAndShift(Scalable, PointerVectorScalableFieldInfo); } } }; Index: lib/CodeGen/LowLevelType.cpp =================================================================== --- lib/CodeGen/LowLevelType.cpp +++ lib/CodeGen/LowLevelType.cpp @@ -21,10 +21,11 @@ LLT llvm::getLLTForType(Type &Ty, const DataLayout &DL) { if (auto VTy = dyn_cast(&Ty)) { auto NumElements = VTy->getNumElements(); + bool Scalable = VTy->isScalable(); LLT ScalarTy = getLLTForType(*VTy->getElementType(), DL); if (NumElements == 1) return ScalarTy; - return LLT::vector(NumElements, ScalarTy); + return LLT::vector(NumElements, ScalarTy, Scalable); } else if (auto PTy = dyn_cast(&Ty)) { return LLT::pointer(PTy->getAddressSpace(), DL.getTypeSizeInBits(&Ty)); } else if (Ty.isSized()) { Index: lib/Support/LowLevelType.cpp =================================================================== --- lib/Support/LowLevelType.cpp +++ lib/Support/LowLevelType.cpp @@ -20,13 +20,13 @@ if (VT.isVector()) { init(/*isPointer=*/false, VT.getVectorNumElements() > 1, VT.getVectorNumElements(), VT.getVectorElementType().getSizeInBits(), - /*AddressSpace=*/0); + /*AddressSpace=*/0, VT.isScalableVector()); } else if (VT.isValid()) { // Aggregates are no different from real scalars as far as GlobalISel is // concerned. assert(VT.getSizeInBits() != 0 && "invalid zero-sized type"); init(/*isPointer=*/false, /*isVector=*/false, /*NumElements=*/0, - VT.getSizeInBits(), /*AddressSpace=*/0); + VT.getSizeInBits(), /*AddressSpace=*/0, /*isScalable=*/false); } else { IsPointer = false; IsVector = false; @@ -35,8 +35,13 @@ } void LLT::print(raw_ostream &OS) const { - if (isVector()) - OS << "<" << getNumElements() << " x " << getElementType() << ">"; + if (isVector()) { + OS << "<"; + if (isScalable()) + OS << "scalable "; + + OS << getNumElements() << " x " << getElementType() << ">"; + } else if (isPointer()) OS << "p" << getAddressSpace(); else if (isValid()) { @@ -50,7 +55,9 @@ const constexpr LLT::BitFieldInfo LLT::PointerSizeFieldInfo; const constexpr LLT::BitFieldInfo LLT::PointerAddressSpaceFieldInfo; const constexpr LLT::BitFieldInfo LLT::VectorElementsFieldInfo; +const constexpr LLT::BitFieldInfo LLT::VectorScalableFieldInfo; const constexpr LLT::BitFieldInfo LLT::VectorSizeFieldInfo; const constexpr LLT::BitFieldInfo LLT::PointerVectorElementsFieldInfo; +const constexpr LLT::BitFieldInfo LLT::PointerVectorScalableFieldInfo; const constexpr LLT::BitFieldInfo LLT::PointerVectorSizeFieldInfo; const constexpr LLT::BitFieldInfo LLT::PointerVectorAddressSpaceFieldInfo; Index: utils/TableGen/GlobalISelEmitter.cpp =================================================================== --- utils/TableGen/GlobalISelEmitter.cpp +++ utils/TableGen/GlobalISelEmitter.cpp @@ -117,7 +117,12 @@ return; } if (Ty.isVector()) { - OS << "GILLT_v" << Ty.getNumElements() << "s" << Ty.getScalarSizeInBits(); + if (Ty.isScalable()) { + OS << "GILLT_nxv"; + } else { + OS << "GILLT_v"; + } + OS << Ty.getNumElements() << "s" << Ty.getScalarSizeInBits(); return; } if (Ty.isPointer()) { @@ -136,7 +141,7 @@ } if (Ty.isVector()) { OS << "LLT::vector(" << Ty.getNumElements() << ", " - << Ty.getScalarSizeInBits() << ")"; + << Ty.getScalarSizeInBits() << ", " << Ty.isScalable() << ")"; return; } if (Ty.isPointer() && Ty.getSizeInBits() > 0) { @@ -164,6 +169,8 @@ return Ty.isScalar() < Other.Ty.isScalar(); if (Ty.isPointer() != Other.Ty.isPointer()) return Ty.isPointer() < Other.Ty.isPointer(); + if (Ty.isVector() && Ty.isScalable() != Other.Ty.isScalable()) + return Ty.isScalable() < Other.Ty.isScalable(); if (Ty.isPointer() && Ty.getAddressSpace() != Other.Ty.getAddressSpace()) return Ty.getAddressSpace() < Other.Ty.getAddressSpace(); @@ -188,7 +195,8 @@ if (VT.isVector() && VT.getVectorNumElements() != 1) return LLTCodeGen( - LLT::vector(VT.getVectorNumElements(), VT.getScalarSizeInBits())); + LLT::vector(VT.getVectorNumElements(), VT.getScalarSizeInBits(), + VT.isScalableVector())); if (VT.isInteger() || VT.isFloatingPoint()) return LLTCodeGen(LLT::scalar(VT.getSizeInBits())); @@ -3500,6 +3508,14 @@ if (!OpTyOrNone) return failedImport("Dst operand has an unsupported type"); + // If the type of a value created as part of a destination pattern is + // not yet known, add it to the list of known types so that it will + // at least be recognized if there are no source patterns which currently + // include that type. + auto Ty = OpTyOrNone.getValue(); + if (KnownTypes.count(Ty) == 0) + KnownTypes.insert(Ty); + unsigned TempRegID = Rule.allocateTempRegID(); InsertPt = Rule.insertAction( InsertPt, OpTyOrNone.getValue(), TempRegID);