[SLP] Calculate overload types from intrinsic description

[sepavloff]

Summary

The previous way to determine overload types of an intrinsic could work
only with functions that have return value of overloaded type. Functions
having return type described with SameVecWidthArgument were not
supported. This change implements overload type determination based on
intrinsic description provided by getIntrinsicInfoTableEntries.

The change does not affect functionality now but represents a
prerequisite for subsequent patches.

[sepavloff]

Any feedback?

[ABataev]

Test?

[bjope]

llvm/include/llvm/IR/Intrinsics.h

193

This did not assert on PtrToElt in the past? Since commit msg says that this does not affect functionality I wonder if this was an intentional change or not?

[RKSimon]

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

5563

Can we pull some of this out into a helper - SLP is far too large already......

Also, won't we need most of this for LoopVectorizer as well?

[spatel]

Is this D109049 without the isnan intrinsic changes?
Is it possible to show a SLP difference with some existing intrinsic with mixed/complex types like @llvm.uadd.with.overflow?

[RKSimon]

In D109484#3004064, @spatel wrote:

Is this D109049 without the isnan intrinsic changes?
Is it possible to show a SLP difference with some existing intrinsic with mixed/complex types like @llvm.uadd.with.overflow?

For reference, this is PR40853

[RKSimon]

In D109484#3004159, @RKSimon wrote:

In D109484#3004064, @spatel wrote:

Is this D109049 without the isnan intrinsic changes?
Is it possible to show a SLP difference with some existing intrinsic with mixed/complex types like @llvm.uadd.with.overflow?

For reference, this is PR40853

And a stretch goal would be PR38424 - vectorizing math calls containing pointers, sincos/modf etc.

llvm/include/llvm/IR/Intrinsics.h

enum ArgKind {
       AK_Any,
       AK_AnyInteger,
       AK_AnyFloat,
       AK_AnyVector,
       AK_AnyPointer,
       AK_MatchType = 7
     };
 
+    /// Returns true if type of this argument or return value depends on the
+    /// type of other argument.
+    bool hasBoundType() const {
+      switch (Kind) {
+      case Argument:
+      case ExtendArgument:
+      case TruncArgument:
+      case HalfVecArgument:
+      case SameVecWidthArgument:
+      case PtrToArgument:
+      case PtrToElt:
+      case VecOfAnyPtrsToElt:
+      case VecElementArgument:
+      case Subdivide2Argument:
+      case Subdivide4Argument:
+      case VecOfBitcastsToInt:
+        return true;
+      default:
+        return false;
+      }
+      llvm_unreachable("unhandled");
+    }
+
     unsigned getArgumentNumber() const {
-      assert(Kind == Argument || Kind == ExtendArgument ||
-             Kind == TruncArgument || Kind == HalfVecArgument ||
-             Kind == SameVecWidthArgument || Kind == PtrToArgument ||
-             Kind == PtrToElt || Kind == VecElementArgument ||
-             Kind == Subdivide2Argument || Kind == Subdivide4Argument ||
-             Kind == VecOfBitcastsToInt);
+      assert(hasBoundType() && Kind != VecOfAnyPtrsToElt &&
+             "This descriptor does not have argument number");
       return Argument_Info >> 3;
     }
     ArgKind getArgumentKind() const {
-      assert(Kind == Argument || Kind == ExtendArgument ||
-             Kind == TruncArgument || Kind == HalfVecArgument ||
-             Kind == SameVecWidthArgument || Kind == PtrToArgument ||
-             Kind == VecElementArgument || Kind == Subdivide2Argument ||
-             Kind == Subdivide4Argument || Kind == VecOfBitcastsToInt);
+      assert(hasBoundType() && Kind != VecOfAnyPtrsToElt &&
+             "This descriptor does not have argument kind");
       return (ArgKind)(Argument_Info & 7);
     }
 
     // VecOfAnyPtrsToElt uses both an overloaded argument (for address space)
     // and a reference argument (for matching vector width and element types)
     unsigned getOverloadArgNumber() const {
       assert(Kind == VecOfAnyPtrsToElt);
       return Argument_Info >> 16;
static IITDescriptor getVector(unsigned Width, bool IsScalable) {
       return Result;
     }
   };
 
   /// Return the IIT table descriptor for the specified intrinsic into an array
   /// of IITDescriptors.
   void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl<IITDescriptor> &T);
 
+  /// Given the IIT table descriptor skips the descriptors corresponding to one
+  /// type element (either return value or an argument).
+  ///
+  /// The function is used to iterate through intrinsic description without
+  /// building types.
+  void popFirstElement(ArrayRef<Intrinsic::IITDescriptor> &Infos);
+
   enum MatchIntrinsicTypesResult {
     MatchIntrinsicTypes_Match = 0,
     MatchIntrinsicTypes_NoMatchRet = 1,
     MatchIntrinsicTypes_NoMatchArg = 2,
   };
 
   /// Match the specified function type with the type constraints specified by
   /// the .td file. If the given type is an overloaded type it is pushed to the

llvm/lib/IR/Function.cpp

void Intrinsic::getIntrinsicInfoTableEntries(ID id,
   }
 
   // Okay, decode the table into the output vector of IITDescriptors.
   DecodeIITType(NextElt, IITEntries, IIT_Done, T);
   while (NextElt != IITEntries.size() && IITEntries[NextElt] != 0)
     DecodeIITType(NextElt, IITEntries, IIT_Done, T);
 }
 
+void Intrinsic::popFirstElement(ArrayRef<Intrinsic::IITDescriptor> &Infos) {
+  assert(!Infos.empty() && "Descriptor array must not be empty");
+
+  IITDescriptor D = Infos.front();
+  Infos = Infos.slice(1);
+  switch (D.Kind) {
+  case IITDescriptor::Void:
+  case IITDescriptor::VarArg:
+  case IITDescriptor::MMX:
+  case IITDescriptor::AMX:
+  case IITDescriptor::Token:
+  case IITDescriptor::Metadata:
+  case IITDescriptor::Half:
+  case IITDescriptor::BFloat:
+  case IITDescriptor::Float:
+  case IITDescriptor::Double:
+  case IITDescriptor::Quad:
+  case IITDescriptor::Integer:
+  case IITDescriptor::Argument:
+  case IITDescriptor::ExtendArgument:
+  case IITDescriptor::TruncArgument:
+  case IITDescriptor::Subdivide2Argument:
+  case IITDescriptor::Subdivide4Argument:
+  case IITDescriptor::HalfVecArgument:
+  case IITDescriptor::PtrToArgument:
+  case IITDescriptor::PtrToElt:
+  case IITDescriptor::VecElementArgument:
+  case IITDescriptor::VecOfBitcastsToInt:
+  case IITDescriptor::VecOfAnyPtrsToElt:
+    return;
+  case IITDescriptor::Vector:
+  case IITDescriptor::Pointer:
+  case IITDescriptor::SameVecWidthArgument:
+    popFirstElement(Infos);
+    return;
+  case IITDescriptor::Struct:
+    for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i)
+      popFirstElement(Infos);
+    return;
+  }
+  llvm_unreachable("Unhandled argument descriptor");
+}
+
 static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
                              ArrayRef<Type*> Tys, LLVMContext &Context) {
   using namespace Intrinsic;
 
   IITDescriptor D = Infos.front();
   Infos = Infos.slice(1);
 
   switch (D.Kind) {

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

case Instruction::Call: {
         IID = FI->getIntrinsicID();
 
       Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
 
       auto VecCallCosts = getVectorCallCosts(CI, VecTy, TTI, TLI);
       bool UseIntrinsic = ID != Intrinsic::not_intrinsic &&
                           VecCallCosts.first <= VecCallCosts.second;
 
+      SmallVector<Intrinsic::IITDescriptor, 8> Table;
+      ArrayRef<Intrinsic::IITDescriptor> TableRef;
+      if (UseIntrinsic) {
+        getIntrinsicInfoTableEntries(ID, Table);
+        TableRef = Table;
+      }
+      SmallVector<Type *, 2> TysForDecl;
+
+      const auto UpdateTysForDecl = [&](Value *V, bool IsScalar) {
+        if (!UseIntrinsic)
+          return;
+        const Intrinsic::IITDescriptor &Item = TableRef.front();
+        switch (Item.Kind) {
+        case Intrinsic::IITDescriptor::Argument:
+          if (Item.getArgumentNumber() >= TysForDecl.size()) {
+            if (IsScalar)
+              TysForDecl.push_back(V->getType());
+            else
+              TysForDecl.push_back(
+                  FixedVectorType::get(V->getType(), E->Scalars.size()));
+          }
+          break;
+        case Intrinsic::IITDescriptor::SameVecWidthArgument:
+          break;
+        default:
+          assert(!Item.hasBoundType());
+        }
+        popFirstElement(TableRef);
+      };
+
+      UpdateTysForDecl(CI, false);
       Value *ScalarArg = nullptr;
       std::vector<Value *> OpVecs;
-      SmallVector<Type *, 2> TysForDecl =
-          {FixedVectorType::get(CI->getType(), E->Scalars.size())};
       for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
-        ValueList OpVL;
+        if (UseIntrinsic) {
           // Some intrinsics have scalar arguments. This argument should not be
           // vectorized.
-        if (UseIntrinsic && hasVectorInstrinsicScalarOpd(IID, j)) {
+          bool IsScalar = hasVectorInstrinsicScalarOpd(IID, j);
+          if (IsScalar) {
             CallInst *CEI = cast<CallInst>(VL0);
             ScalarArg = CEI->getArgOperand(j);
             OpVecs.push_back(CEI->getArgOperand(j));
-          if (hasVectorInstrinsicOverloadedScalarOpd(IID, j))
-            TysForDecl.push_back(ScalarArg->getType());
+          }
+          UpdateTysForDecl(CI->getArgOperand(j), IsScalar);
+          if (IsScalar)
             continue;
         }
 
         Value *OpVec = vectorizeTree(E->getOperand(j));
         LLVM_DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n");
         OpVecs.push_back(OpVec);
       }
 
       Function *CF;