[SVE][TableGen] LLT support for scalable vectors

[huntergr]

Summary

Adds a 'Scalable' bit in the raw_data area for LLT vector types,
and adjusts constructors, printers, etc. to recognize that.

Also ensures that types used in destination patterns in the
GlobalISelEmitter are added to the list of known types even if
they are not present in source patterns.

Part of the initial SVE codegen series, rfc will be posted soon.

[huntergr]

This comment was removed by huntergr.

[aemerson]

We can take a look at this once SVE fully lands in llvm. Resigning to get it off my dashboard until then.

include/llvm/Support/LowLevelTypeImpl.h

 class raw_ostream;
 
 class LLT {
 public:
   /// Get a low-level scalar or aggregate "bag of bits".
   static LLT scalar(unsigned SizeInBits) {
     assert(SizeInBits > 0 && "invalid scalar size");
     return LLT{/*isPointer=*/false, /*isVector=*/false, /*NumElements=*/0,
-               SizeInBits, /*AddressSpace=*/0};
+               SizeInBits, /*AddressSpace=*/0, /*isScalable=*/false};
   }
 
   /// Get a low-level pointer in the given address space (defaulting to 0).
   static LLT pointer(uint16_t AddressSpace, unsigned SizeInBits) {
     assert(SizeInBits > 0 && "invalid pointer size");
     return LLT{/*isPointer=*/true, /*isVector=*/false, /*NumElements=*/0,
-               SizeInBits, AddressSpace};
+               SizeInBits, AddressSpace, /*isScalable=*/false};
   }
 
   /// Get a low-level vector of some number of elements and element width.
   /// \p NumElements must be at least 2.
-  static LLT vector(uint16_t NumElements, unsigned ScalarSizeInBits) {
+  static LLT vector(uint16_t NumElements, unsigned ScalarSizeInBits,
+                    bool Scalable = false) {
     assert(NumElements > 1 && "invalid number of vector elements");
     assert(ScalarSizeInBits > 0 && "invalid vector element size");
     return LLT{/*isPointer=*/false, /*isVector=*/true, NumElements,
-               ScalarSizeInBits, /*AddressSpace=*/0};
+               ScalarSizeInBits, /*AddressSpace=*/0, Scalable};
   }
 
   /// Get a low-level vector of some number of elements and element type.
-  static LLT vector(uint16_t NumElements, LLT ScalarTy) {
+  static LLT vector(uint16_t NumElements, LLT ScalarTy, bool Scalable = false) {
     assert(NumElements > 1 && "invalid number of vector elements");
     assert(!ScalarTy.isVector() && "invalid vector element type");
     return LLT{ScalarTy.isPointer(), /*isVector=*/true, NumElements,
                ScalarTy.getSizeInBits(),
-               ScalarTy.isPointer() ? ScalarTy.getAddressSpace() : 0};
+               ScalarTy.isPointer() ? ScalarTy.getAddressSpace() : 0,
+               Scalable};
   }
 
   explicit LLT(bool isPointer, bool isVector, uint16_t NumElements,
-               unsigned SizeInBits, unsigned AddressSpace) {
-    init(isPointer, isVector, NumElements, SizeInBits, AddressSpace);
+               unsigned SizeInBits, unsigned AddressSpace, bool Scalable) {
+    init(isPointer, isVector, NumElements, SizeInBits, AddressSpace, Scalable);
   }
   explicit LLT() : IsPointer(false), IsVector(false), RawData(0) {}
 
   explicit LLT(MVT VT);
 
   bool isValid() const { return RawData != 0; }
 
   bool isScalar() const { return isValid() && !IsPointer && !IsVector; }
public:
   LLT getElementType() const {
     assert(isVector() && "cannot get element type of scalar/aggregate");
     if (IsPointer)
       return pointer(getAddressSpace(), getScalarSizeInBits());
     else
       return scalar(getScalarSizeInBits());
   }
 
+  bool isScalable() const {
+    assert(isVector() && "cannot get scalable property of non-vector");
+    if (IsPointer)
+      return (getFieldValue(PointerVectorScalableFieldInfo) != 0);
+    else
+      return (getFieldValue(VectorScalableFieldInfo) != 0);
+  }
+
   void print(raw_ostream &OS) const;
 
   bool operator==(const LLT &RHS) const {
     return IsPointer == RHS.IsPointer && IsVector == RHS.IsVector &&
            RHS.RawData == RawData;
   }
 
   bool operator!=(const LLT &RHS) const { return !(*this == RHS); }
private:
   ///   SizeInBits: 16;
   ///   AddressSpace: 23;
   static const constexpr BitFieldInfo PointerSizeFieldInfo{16, 0};
   static const constexpr BitFieldInfo PointerAddressSpaceFieldInfo{
       23, PointerSizeFieldInfo[0] + PointerSizeFieldInfo[1]};
   /// * Vector-of-non-pointer (isPointer == 0 && isVector == 1):
   ///   NumElements: 16;
   ///   SizeOfElement: 32;
+  ///   Scalable: 1;
   static const constexpr BitFieldInfo VectorElementsFieldInfo{16, 0};
   static const constexpr BitFieldInfo VectorSizeFieldInfo{
       32, VectorElementsFieldInfo[0] + VectorElementsFieldInfo[1]};
+  static const constexpr BitFieldInfo VectorScalableFieldInfo{
+      1, VectorSizeFieldInfo[0] + VectorSizeFieldInfo[1]};
   /// * Vector-of-pointer (isPointer == 1 && isVector == 1):
   ///   NumElements: 16;
   ///   SizeOfElement: 16;
   ///   AddressSpace: 23;
+  ///   Scalable: 1;
   static const constexpr BitFieldInfo PointerVectorElementsFieldInfo{16, 0};
   static const constexpr BitFieldInfo PointerVectorSizeFieldInfo{
       16,
       PointerVectorElementsFieldInfo[1] + PointerVectorElementsFieldInfo[0]};
   static const constexpr BitFieldInfo PointerVectorAddressSpaceFieldInfo{
       23, PointerVectorSizeFieldInfo[1] + PointerVectorSizeFieldInfo[0]};
+  static const constexpr BitFieldInfo PointerVectorScalableFieldInfo{
+      1, PointerVectorAddressSpaceFieldInfo[0] +
+         PointerVectorAddressSpaceFieldInfo[1]};
 
   uint64_t IsPointer : 1;
   uint64_t IsVector : 1;
   uint64_t RawData : 62;
 
   static uint64_t getMask(const BitFieldInfo FieldInfo) {
     const int FieldSizeInBits = FieldInfo[0];
     return (((uint64_t)1) << FieldSizeInBits) - 1;
   }
   static uint64_t maskAndShift(uint64_t Val, uint64_t Mask, uint8_t Shift) {
     assert(Val <= Mask && "Value too large for field");
     return (Val & Mask) << Shift;
   }
   static uint64_t maskAndShift(uint64_t Val, const BitFieldInfo FieldInfo) {
     return maskAndShift(Val, getMask(FieldInfo), FieldInfo[1]);
   }
   uint64_t getFieldValue(const BitFieldInfo FieldInfo) const {
     return getMask(FieldInfo) & (RawData >> FieldInfo[1]);
   }
 
   void init(bool IsPointer, bool IsVector, uint16_t NumElements,
-            unsigned SizeInBits, unsigned AddressSpace) {
+            unsigned SizeInBits, unsigned AddressSpace, bool IsScalable) {
     this->IsPointer = IsPointer;
     this->IsVector = IsVector;
+    uint64_t Scalable = IsScalable ? 1 : 0;
     if (!IsVector) {
       if (!IsPointer)
         RawData = maskAndShift(SizeInBits, ScalarSizeFieldInfo);
       else
         RawData = maskAndShift(SizeInBits, PointerSizeFieldInfo) |
                   maskAndShift(AddressSpace, PointerAddressSpaceFieldInfo);
     } else {
       assert(NumElements > 1 && "invalid number of vector elements");
       if (!IsPointer)
         RawData = maskAndShift(NumElements, VectorElementsFieldInfo) |
-                  maskAndShift(SizeInBits, VectorSizeFieldInfo);
+                  maskAndShift(SizeInBits, VectorSizeFieldInfo) |
+                  maskAndShift(Scalable, VectorScalableFieldInfo);
       else
         RawData =
             maskAndShift(NumElements, PointerVectorElementsFieldInfo) |
             maskAndShift(SizeInBits, PointerVectorSizeFieldInfo) |
-            maskAndShift(AddressSpace, PointerVectorAddressSpaceFieldInfo);
+            maskAndShift(AddressSpace, PointerVectorAddressSpaceFieldInfo) |
+            maskAndShift(Scalable, PointerVectorScalableFieldInfo);
     }
   }
 };
 
 inline raw_ostream& operator<<(raw_ostream &OS, const LLT &Ty) {
   Ty.print(OS);
   return OS;
 }

lib/CodeGen/LowLevelType.cpp

 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
 LLT llvm::getLLTForType(Type &Ty, const DataLayout &DL) {
   if (auto VTy = dyn_cast<VectorType>(&Ty)) {
     auto NumElements = VTy->getNumElements();
+    bool Scalable = VTy->isScalable();
     LLT ScalarTy = getLLTForType(*VTy->getElementType(), DL);
     if (NumElements == 1)
       return ScalarTy;
-    return LLT::vector(NumElements, ScalarTy);
+    return LLT::vector(NumElements, ScalarTy, Scalable);
   } else if (auto PTy = dyn_cast<PointerType>(&Ty)) {
     return LLT::pointer(PTy->getAddressSpace(), DL.getTypeSizeInBits(&Ty));
   } else if (Ty.isSized()) {
     // Aggregates are no different from real scalars as far as GlobalISel is
     // concerned.
     auto SizeInBits = DL.getTypeSizeInBits(&Ty);
     assert(SizeInBits != 0 && "invalid zero-sized type");
     return LLT::scalar(SizeInBits);
   }
   return LLT();
 }

lib/Support/LowLevelType.cpp

 #include "llvm/Support/LowLevelTypeImpl.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
 LLT::LLT(MVT VT) {
   if (VT.isVector()) {
     init(/*isPointer=*/false, VT.getVectorNumElements() > 1,
          VT.getVectorNumElements(), VT.getVectorElementType().getSizeInBits(),
-         /*AddressSpace=*/0);
+         /*AddressSpace=*/0, VT.isScalableVector());
   } else if (VT.isValid()) {
     // Aggregates are no different from real scalars as far as GlobalISel is
     // concerned.
     assert(VT.getSizeInBits() != 0 && "invalid zero-sized type");
     init(/*isPointer=*/false, /*isVector=*/false, /*NumElements=*/0,
-         VT.getSizeInBits(), /*AddressSpace=*/0);
+         VT.getSizeInBits(), /*AddressSpace=*/0, /*isScalable=*/false);
   } else {
     IsPointer = false;
     IsVector = false;
     RawData = 0;
   }
 }
 
 void LLT::print(raw_ostream &OS) const {
-  if (isVector())
-    OS << "<" << getNumElements() << " x " << getElementType() << ">";
+  if (isVector()) {
+    OS << "<";
+    if (isScalable())
+      OS << "scalable ";
+
+    OS << getNumElements() << " x " << getElementType() << ">";
+  }
   else if (isPointer())
     OS << "p" << getAddressSpace();
   else if (isValid()) {
     assert(isScalar() && "unexpected type");
     OS << "s" << getScalarSizeInBits();
   } else
     OS << "LLT_invalid";
 }
 
 const constexpr LLT::BitFieldInfo LLT::ScalarSizeFieldInfo;
 const constexpr LLT::BitFieldInfo LLT::PointerSizeFieldInfo;
 const constexpr LLT::BitFieldInfo LLT::PointerAddressSpaceFieldInfo;
 const constexpr LLT::BitFieldInfo LLT::VectorElementsFieldInfo;
+const constexpr LLT::BitFieldInfo LLT::VectorScalableFieldInfo;
 const constexpr LLT::BitFieldInfo LLT::VectorSizeFieldInfo;
 const constexpr LLT::BitFieldInfo LLT::PointerVectorElementsFieldInfo;
+const constexpr LLT::BitFieldInfo LLT::PointerVectorScalableFieldInfo;
 const constexpr LLT::BitFieldInfo LLT::PointerVectorSizeFieldInfo;
 const constexpr LLT::BitFieldInfo LLT::PointerVectorAddressSpaceFieldInfo;

utils/TableGen/GlobalISelEmitter.cpp

public:
   }
 
   void emitCxxEnumValue(raw_ostream &OS) const {
     if (Ty.isScalar()) {
       OS << "GILLT_s" << Ty.getSizeInBits();
       return;
     }
     if (Ty.isVector()) {
-      OS << "GILLT_v" << Ty.getNumElements() << "s" << Ty.getScalarSizeInBits();
+      if (Ty.isScalable()) {
+        OS << "GILLT_nxv";
+      } else {
+        OS << "GILLT_v";
+      }
+      OS << Ty.getNumElements() << "s" << Ty.getScalarSizeInBits();
       return;
     }
     if (Ty.isPointer()) {
       OS << "GILLT_p" << Ty.getAddressSpace();
       if (Ty.getSizeInBits() > 0)
         OS << "s" << Ty.getSizeInBits();
       return;
     }
     llvm_unreachable("Unhandled LLT");
   }
 
   void emitCxxConstructorCall(raw_ostream &OS) const {
     if (Ty.isScalar()) {
       OS << "LLT::scalar(" << Ty.getSizeInBits() << ")";
       return;
     }
     if (Ty.isVector()) {
       OS << "LLT::vector(" << Ty.getNumElements() << ", "
-         << Ty.getScalarSizeInBits() << ")";
+         << Ty.getScalarSizeInBits() << ", " << Ty.isScalable() << ")";
       return;
     }
     if (Ty.isPointer() && Ty.getSizeInBits() > 0) {
       OS << "LLT::pointer(" << Ty.getAddressSpace() << ", "
          << Ty.getSizeInBits() << ")";
       return;
     }
     llvm_unreachable("Unhandled LLT");
if (!Ty.isValid())
       return false;
 
     if (Ty.isVector() != Other.Ty.isVector())
       return Ty.isVector() < Other.Ty.isVector();
     if (Ty.isScalar() != Other.Ty.isScalar())
       return Ty.isScalar() < Other.Ty.isScalar();
     if (Ty.isPointer() != Other.Ty.isPointer())
       return Ty.isPointer() < Other.Ty.isPointer();
+    if (Ty.isVector() && Ty.isScalable() != Other.Ty.isScalable())
+      return Ty.isScalable() < Other.Ty.isScalable();
 
     if (Ty.isPointer() && Ty.getAddressSpace() != Other.Ty.getAddressSpace())
       return Ty.getAddressSpace() < Other.Ty.getAddressSpace();
 
     if (Ty.isVector() && Ty.getNumElements() != Other.Ty.getNumElements())
       return Ty.getNumElements() < Other.Ty.getNumElements();
 
     return Ty.getSizeInBits() < Other.Ty.getSizeInBits();
   }
 
   bool operator==(const LLTCodeGen &B) const { return Ty == B.Ty; }
 };
 
 // Track all types that are used so we can emit the corresponding enum.
 std::set<LLTCodeGen> KnownTypes;
 
 class InstructionMatcher;
 /// Convert an MVT to an equivalent LLT if possible, or the invalid LLT() for
 /// MVTs that don't map cleanly to an LLT (e.g., iPTR, *any, ...).
 static Optional<LLTCodeGen> MVTToLLT(MVT::SimpleValueType SVT) {
   MVT VT(SVT);
 
   if (VT.isVector() && VT.getVectorNumElements() != 1)
     return LLTCodeGen(
-        LLT::vector(VT.getVectorNumElements(), VT.getScalarSizeInBits()));
+        LLT::vector(VT.getVectorNumElements(), VT.getScalarSizeInBits(),
+                    VT.isScalableVector()));
 
   if (VT.isInteger() || VT.isFloatingPoint())
     return LLTCodeGen(LLT::scalar(VT.getSizeInBits()));
   return None;
 }
 
 static std::string explainPredicates(const TreePatternNode *N) {
   std::string Explanation = "";
if (DstChild->getOperator()->isSubClassOf("Instruction")) {
 
       Optional<LLTCodeGen> OpTyOrNone = None;
       if (ChildTypes.front().isMachineValueType())
         OpTyOrNone =
             MVTToLLT(ChildTypes.front().getMachineValueType().SimpleTy);
       if (!OpTyOrNone)
         return failedImport("Dst operand has an unsupported type");
 
+      // If the type of a value created as part of a destination pattern is
+      // not yet known, add it to the list of known types so that it will
+      // at least be recognized if there are no source patterns which currently
+      // include that type.
+      auto Ty = OpTyOrNone.getValue();
+      if (KnownTypes.count(Ty) == 0)
+        KnownTypes.insert(Ty);
+
       unsigned TempRegID = Rule.allocateTempRegID();
       InsertPt = Rule.insertAction<MakeTempRegisterAction>(
           InsertPt, OpTyOrNone.getValue(), TempRegID);
       DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID);
 
       auto InsertPtOrError = createAndImportSubInstructionRenderer(
           ++InsertPt, Rule, DstChild, TempRegID);
       if (auto Error = InsertPtOrError.takeError())