[IntrinsicEmitter] Add overloaded types for SVE intrinsics (Subdivide2 & Subdivide4)

[kmclaughlin]

Summary

Both match the type of another intrinsic parameter of a vector type, but where each element is subdivided to form a vector with more elements of a smaller type.

Subdivide2Argument allows intrinsics such as the following to be defined:

• declare <vscale x 4 x i32> @llvm.something.nxv4i32(<vscale x 8 x i16>)

Subdivide4Argument allows intrinsics such as:

• declare <vscale x 4 x i32> @llvm.something.nxv4i32(<vscale x 16 x i8>)

Tests are included in follow up patches which add intrinsics using these types.

[rovka]

include/llvm/IR/DerivedTypes.h

531

Why not move this logic to getTruncatedElementVectorType and drop getNarrowerFpElementVectorType altogether?

include/llvm/IR/Intrinsics.h

130

Not directly related to your change: Is there a reason why VecOfAnyPtrsToElt isn't in this assert? If not, maybe this is a good time to add a First/LastArgumentKind and just check that Kind is in the range.

lib/IR/Function.cpp

989

Maybe replace this with an assert and drop the unreachable.

997

Ditto,

1309

Can you share this code? Either a small helper or a fallthrough with a repeat check on D.Kind to get the number of subdivisions would work.

[sdesmalen]

include/llvm/IR/DerivedTypes.h

516

nit: Use llvm_unreachable("message") instead of assert(0 && "message")

[kmclaughlin]

• Moved getNarrowerFpElementVectorType logic into getTruncatedElementVectorType
• Shared code which handles Subdivide2Argument and Subdivide4Argument where possible
• Replaced an unreachable with an assert in Function.cpp
• Replaced an assert with an unreachable in DerivedTypes.h

[kmclaughlin]

Thanks for reviewing this patch, @rovka and @sdesmalen!

include/llvm/IR/Intrinsics.h

130

I think it's probably best not to include VecOfAnyPtrsToElt here as it was removed from the assert when the functions getOverloadArgNumber and getRefArgNumber below were added (these should be used instead of getArgumentNumber in this case).

lib/IR/Function.cpp

1309

This code is now shared, as is the code to handle Subdivide2Argument & Subdivide4Argument in Function.cpp above.

[sdesmalen]

Thanks for the changes @kmclaughlin! Just a few more nits from me, but looks good otherwise.

include/llvm/IR/DerivedTypes.h

493

nit: I don't think the break is needed because of the llvm_unreachable

lib/IR/Function.cpp

992

nit: you can simplify this by:

int SubDiv = D.Kind == IITDescriptor::Subdivide2Argument ? 1 : 2;
return VectorType::getSubdividedVectorType(VTy, SubDiv);

1302

nit: you can use auto *VTy here since the dyn_cast<VectorType> makes it clear that the return type is VectorType.

1303

nit: can be simplified by:

int SubDiv = D.Kind == IITDescriptor::Subdivide2Argument ? 1 : 2;
NewTy = VectorType::getSubdividedVectorType(VTy, SubDiv);

1310

nit: Move this return statement into the if(VectorType *VTy = dyn_cast<VectorType>(NewTy)) block?

[kmclaughlin]

• Some minor changes, including removing an unnecessary break
• Simplified checks of D.Kind in Function.cpp to determine if it is a Subdivide2Argument or Subdivide4Argument

[sdesmalen]

Thanks, LGTM!

include/llvm/IR/DerivedTypes.h

public:
   /// This static method is like getInteger except that the element types are
   /// twice as wide as the elements in the input type.
   static VectorType *getExtendedElementVectorType(VectorType *VTy) {
     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
     Type *EltTy = IntegerType::get(VTy->getContext(), EltBits * 2);
     return VectorType::get(EltTy, VTy->getElementCount());
   }
 
-  /// This static method is like getInteger except that the element types are
-  /// half as wide as the elements in the input type.
+  // This static method gets a VectorType with the same number of elements as
+  // the input type, and the element type is an integer or float type which
+  // is half as wide as the elements in the input type.
   static VectorType *getTruncatedElementVectorType(VectorType *VTy) {
+    Type *EltTy;
+    if (VTy->getElementType()->isFloatingPointTy()) {
+      switch(VTy->getElementType()->getTypeID()) {
+      case DoubleTyID:
+        EltTy = Type::getFloatTy(VTy->getContext());
+        break;
+      case FloatTyID:
+        EltTy = Type::getHalfTy(VTy->getContext());
+        break;
+      default:
+        llvm_unreachable("Cannot create narrower fp vector element type");
+        break;
+      }
+    } else {
       unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
       assert((EltBits & 1) == 0 &&
              "Cannot truncate vector element with odd bit-width");
-    Type *EltTy = IntegerType::get(VTy->getContext(), EltBits / 2);
+      EltTy = IntegerType::get(VTy->getContext(), EltBits / 2);
+    }
     return VectorType::get(EltTy, VTy->getElementCount());
   }
 
+  // This static method returns a VectorType with a smaller number of elements
+  // of a larger type than the input element type. For example, a <16 x i8>
+  // subdivided twice would return <4 x i32>
+  static VectorType *getSubdividedVectorType(VectorType *VTy, int NumSubdivs) {
+    for (int i = 0; i < NumSubdivs; ++i) {
+      VTy = VectorType::getDoubleElementsVectorType(VTy);
+      VTy = VectorType::getTruncatedElementVectorType(VTy);
+    }
+    return VTy;
+  }
+
   /// This static method returns a VectorType with half as many elements as the
   /// input type and the same element type.
   static VectorType *getHalfElementsVectorType(VectorType *VTy) {
     auto EltCnt = VTy->getElementCount();
     assert ((EltCnt.Min & 1) == 0 &&
             "Cannot halve vector with odd number of elements.");
     return VectorType::get(VTy->getElementType(), EltCnt/2);
   }
 
   /// This static method returns a VectorType with twice as many elements as the
   /// input type and the same element type.
   static VectorType *getDoubleElementsVectorType(VectorType *VTy) {
     auto EltCnt = VTy->getElementCount();
     assert((VTy->getNumElements() * 2ull) <= UINT_MAX &&
            "Too many elements in vector");
     return VectorType::get(VTy->getElementType(), EltCnt*2);
   }
 
   /// Return true if the specified type is valid as a element type.
   static bool isValidElementType(Type *ElemTy);
 
   /// Return an ElementCount instance to represent the (possibly scalable)
   /// number of elements in the vector.
   ElementCount getElementCount() const {
     uint64_t MinimumEltCnt = getNumElements();

include/llvm/IR/Intrinsics.h

#undef GET_INTRINSIC_ENUM_VALUES
   /// This is a type descriptor which explains the type requirements of an
   /// intrinsic. This is returned by getIntrinsicInfoTableEntries.
   struct IITDescriptor {
     enum IITDescriptorKind {
       Void, VarArg, MMX, Token, Metadata, Half, Float, Double, Quad,
       Integer, Vector, Pointer, Struct,
       Argument, ExtendArgument, TruncArgument, HalfVecArgument,
       SameVecWidthArgument, PtrToArgument, PtrToElt, VecOfAnyPtrsToElt,
-      VecElementArgument, ScalableVecArgument
+      VecElementArgument, ScalableVecArgument, Subdivide2Argument,
+      Subdivide4Argument
     } Kind;
 
     union {
       unsigned Integer_Width;
       unsigned Float_Width;
       unsigned Vector_Width;
       unsigned Pointer_AddressSpace;
       unsigned Struct_NumElements;
       unsigned Argument_Info;
     };
 
     enum ArgKind {
       AK_Any,
       AK_AnyInteger,
       AK_AnyFloat,
       AK_AnyVector,
       AK_AnyPointer,
       AK_MatchType = 7
     };
 
     unsigned getArgumentNumber() const {
       assert(Kind == Argument || Kind == ExtendArgument ||
              Kind == TruncArgument || Kind == HalfVecArgument ||
              Kind == SameVecWidthArgument || Kind == PtrToArgument ||
-             Kind == PtrToElt || Kind == VecElementArgument);
+             Kind == PtrToElt || Kind == VecElementArgument ||
+             Kind == Subdivide2Argument || Kind == Subdivide4Argument);
       return Argument_Info >> 3;
     }
     ArgKind getArgumentKind() const {
       assert(Kind == Argument || Kind == ExtendArgument ||
              Kind == TruncArgument || Kind == HalfVecArgument ||
              Kind == SameVecWidthArgument || Kind == PtrToArgument ||
-             Kind == VecElementArgument);
+             Kind == VecElementArgument || Kind == Subdivide2Argument ||
+             Kind == Subdivide4Argument);
       return (ArgKind)(Argument_Info & 7);
     }
 
     // VecOfAnyPtrsToElt uses both an overloaded argument (for address space)
     // and a reference argument (for matching vector width and element types)
     unsigned getOverloadArgNumber() const {
       assert(Kind == VecOfAnyPtrsToElt);
       return Argument_Info >> 16;

include/llvm/IR/Intrinsics.td

 class LLVMPointerToElt<int num> : LLVMMatchType<num>;
 class LLVMVectorOfAnyPointersToElt<int num> : LLVMMatchType<num>;
 class LLVMVectorElementType<int num> : LLVMMatchType<num>;
 
 // Match the type of another intrinsic parameter that is expected to be a
 // vector type, but change the element count to be half as many
 class LLVMHalfElementsVectorType<int num> : LLVMMatchType<num>;
 
+// Match the type of another intrinsic parameter that is expected to be a
+// vector type (i.e. <N x iM>) but with each element subdivided to
+// form a vector with more elements that are smaller than the original.
+class LLVMSubdivide2VectorType<int num> : LLVMMatchType<num>;
+class LLVMSubdivide4VectorType<int num> : LLVMMatchType<num>;
+
 def llvm_void_ty       : LLVMType<isVoid>;
 let isAny = 1 in {
   def llvm_any_ty        : LLVMType<Any>;
   def llvm_anyint_ty     : LLVMType<iAny>;
   def llvm_anyfloat_ty   : LLVMType<fAny>;
   def llvm_anyvector_ty  : LLVMType<vAny>;
 }
 def llvm_i1_ty         : LLVMType<i1>;

lib/IR/Function.cpp

enum IIT_Info {
   IIT_I128 = 35,
   IIT_V512 = 36,
   IIT_V1024 = 37,
   IIT_STRUCT6 = 38,
   IIT_STRUCT7 = 39,
   IIT_STRUCT8 = 40,
   IIT_F128 = 41,
   IIT_VEC_ELEMENT = 42,
-  IIT_SCALABLE_VEC = 43
+  IIT_SCALABLE_VEC = 43,
+  IIT_SUBDIVIDE2_ARG = 44,
+  IIT_SUBDIVIDE4_ARG = 45
 };
 
 static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
                       SmallVectorImpl<Intrinsic::IITDescriptor> &OutputTable) {
   using namespace Intrinsic;
 
   IIT_Info Info = IIT_Info(Infos[NextElt++]);
   unsigned StructElts = 2;
static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
   case IIT_STRUCT3: ++StructElts; LLVM_FALLTHROUGH;
   case IIT_STRUCT2: {
     OutputTable.push_back(IITDescriptor::get(IITDescriptor::Struct,StructElts));
 
     for (unsigned i = 0; i != StructElts; ++i)
       DecodeIITType(NextElt, Infos, OutputTable);
     return;
   }
+  case IIT_SUBDIVIDE2_ARG: {
+    unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
+    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Subdivide2Argument,
+                                             ArgInfo));
+    return;
+  }
+  case IIT_SUBDIVIDE4_ARG: {
+    unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
+    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Subdivide4Argument,
+                                             ArgInfo));
+    return;
+  }
   case IIT_VEC_ELEMENT: {
     unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
     OutputTable.push_back(IITDescriptor::get(IITDescriptor::VecElementArgument,
                                              ArgInfo));
     return;
   }
   case IIT_SCALABLE_VEC: {
     OutputTable.push_back(IITDescriptor::get(IITDescriptor::ScalableVecArgument,
case IITDescriptor::TruncArgument: {
     Type *Ty = Tys[D.getArgumentNumber()];
     if (VectorType *VTy = dyn_cast<VectorType>(Ty))
       return VectorType::getTruncatedElementVectorType(VTy);
 
     IntegerType *ITy = cast<IntegerType>(Ty);
     assert(ITy->getBitWidth() % 2 == 0);
     return IntegerType::get(Context, ITy->getBitWidth() / 2);
   }
+  case IITDescriptor::Subdivide2Argument:
+  case IITDescriptor::Subdivide4Argument: {
+    Type *Ty = Tys[D.getArgumentNumber()];
+    VectorType *VTy = dyn_cast<VectorType>(Ty);
+    assert(VTy && "Expected an argument of Vector Type");
+    if (D.Kind == IITDescriptor::Subdivide2Argument)
+      return VectorType::getSubdividedVectorType(VTy, 1);
+    return VectorType::getSubdividedVectorType(VTy, 2);
+  }
   case IITDescriptor::HalfVecArgument:
     return VectorType::getHalfElementsVectorType(cast<VectorType>(
                                                   Tys[D.getArgumentNumber()]));
   case IITDescriptor::SameVecWidthArgument: {
     Type *EltTy = DecodeFixedType(Infos, Tys, Context);
     Type *Ty = Tys[D.getArgumentNumber()];
     if (auto *VTy = dyn_cast<VectorType>(Ty))
       return VectorType::get(EltTy, VTy->getElementCount());
     return EltTy;
   }
case IITDescriptor::VecOfAnyPtrsToElt: {
              ReferenceType->getVectorElementType();
     }
     case IITDescriptor::VecElementArgument: {
       if (D.getArgumentNumber() >= ArgTys.size())
         return IsDeferredCheck ? true : DeferCheck(Ty);
       auto *ReferenceType = dyn_cast<VectorType>(ArgTys[D.getArgumentNumber()]);
       return !ReferenceType || Ty != ReferenceType->getElementType();
     }
+    case IITDescriptor::Subdivide2Argument:
+    case IITDescriptor::Subdivide4Argument: {
+      // If this is a forward reference, defer the check for later.
+      if (D.getArgumentNumber() >= ArgTys.size())
+        return IsDeferredCheck || DeferCheck(Ty);
+
+      Type *NewTy = ArgTys[D.getArgumentNumber()];
+      if (VectorType *VTy = dyn_cast<VectorType>(NewTy)) {
+        if (D.Kind == IITDescriptor::Subdivide2Argument)
+          NewTy = VectorType::getSubdividedVectorType(VTy, 1);
+        else
+          NewTy = VectorType::getSubdividedVectorType(VTy, 2);
+      } else
+        return true;
+
+      return Ty != NewTy;
+    }
     case IITDescriptor::ScalableVecArgument: {
       VectorType *VTy = dyn_cast<VectorType>(Ty);
       if (!VTy || !VTy->isScalable())
         return true;
       return matchIntrinsicType(VTy, Infos, ArgTys, DeferredChecks,
                                 IsDeferredCheck);
     }
   }

utils/TableGen/IntrinsicEmitter.cpp

enum IIT_Info {
   IIT_I128 = 35,
   IIT_V512 = 36,
   IIT_V1024 = 37,
   IIT_STRUCT6 = 38,
   IIT_STRUCT7 = 39,
   IIT_STRUCT8 = 40,
   IIT_F128 = 41,
   IIT_VEC_ELEMENT = 42,
-  IIT_SCALABLE_VEC = 43
+  IIT_SCALABLE_VEC = 43,
+  IIT_SUBDIVIDE2_ARG = 44,
+  IIT_SUBDIVIDE4_ARG = 45
 };
 
 static void EncodeFixedValueType(MVT::SimpleValueType VT,
                                  std::vector<unsigned char> &Sig) {
   if (MVT(VT).isInteger()) {
     unsigned BitWidth = MVT(VT).getSizeInBits();
     switch (BitWidth) {
     default: PrintFatalError("unhandled integer type width in intrinsic!");
else if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) {
       Sig.push_back(NextArgCode++);
       // Encode LLVMMatchType<Number> ArgNo
       Sig.push_back(Number);
       return;
     } else if (R->isSubClassOf("LLVMPointerToElt"))
       Sig.push_back(IIT_PTR_TO_ELT);
     else if (R->isSubClassOf("LLVMVectorElementType"))
       Sig.push_back(IIT_VEC_ELEMENT);
+    else if (R->isSubClassOf("LLVMSubdivide2VectorType"))
+      Sig.push_back(IIT_SUBDIVIDE2_ARG);
+    else if (R->isSubClassOf("LLVMSubdivide4VectorType"))
+      Sig.push_back(IIT_SUBDIVIDE4_ARG);
     else
       Sig.push_back(IIT_ARG);
     return Sig.push_back((Number << 3) | 7 /*IITDescriptor::AK_MatchType*/);
   }
 
   MVT::SimpleValueType VT = getValueType(R->getValueAsDef("VT"));
 
   unsigned Tmp = 0;