[CostModel] Align the cost model for intrinsics for scalable/fixed-width vectors.

[sdesmalen]

Summary

Let getIntrinsicInstrCost call getTypeBasedIntrinsicInstrCost for scalable vectors,
similar to how this is done for fixed-width vectors, instead of falling back
on BaseT::getIntrinsicInstrCost().

If the intrinsic cannot be costed (or is not overloaded by the target),
it will return InstructionCost::getInvalid() instead.

Depends on D97469

[dmgreen]

llvm/include/llvm/CodeGen/BasicTTIImpl.h

1345

Should this ScalarizationCost and IntrinsicCostAttributes::ScalarizationCost be converted to InstructionCost too?

[sdesmalen]

llvm/include/llvm/CodeGen/BasicTTIImpl.h

1345

Yes, I have another patch for that which updates the interface of getScalarizationOverhead to return InstructionCost at which points this needs to become an InstructionCost as well. I haven't posted this patch yet, but will do so soon.

[sdesmalen]

Rebased patch.

[david-arm]

LGTM!

llvm/include/llvm/CodeGen/BasicTTIImpl.h

1346

nit: Could you just use RetVF.isFixedLengthVector() instead here?

1392

nit: Could you address the formatting issues?

1809

nit: Formatting issues

[sdesmalen]

Thanks for the review comments!

llvm/include/llvm/CodeGen/BasicTTIImpl.h

1346

That's a method of EVT, not ElementCount unfortunately.

llvm/include/llvm/CodeGen/BasicTTIImpl.h

 //===- BasicTTIImpl.h -------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 /// \file
case Intrinsic::fshr: {
                                         CmpInst::BAD_ICMP_PREDICATE, CostKind);
         Cost +=
             thisT()->getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,
                                         CmpInst::BAD_ICMP_PREDICATE, CostKind);
       }
       return Cost;
     }
     }
-    // TODO: Handle the remaining intrinsic with scalable vector type
-    if (isa<ScalableVectorType>(RetTy))
-      return BaseT::getIntrinsicInstrCost(ICA, CostKind);
 
     // Assume that we need to scalarize this intrinsic.
     // Compute the scalarization overhead based on Args for a vector
     // intrinsic.
     unsigned ScalarizationCost = std::numeric_limits<unsigned>::max();
-    if (RetVF.isVector()) {
+    if (RetVF.isVector() && !RetVF.isScalable()) {
       ScalarizationCost = 0;
       if (!RetTy->isVoidTy())
         ScalarizationCost +=
             getScalarizationOverhead(cast<VectorType>(RetTy), true, false);
       ScalarizationCost +=
           getOperandsScalarizationOverhead(Args, ICA.getArgTypes());
     }
 
if (!Tys.empty()) {
       VecOpTy = dyn_cast<VectorType>(Tys[VecTyIndex]);
     }
 
     // Library call cost - other than size, make it expensive.
     unsigned SingleCallCost = CostKind == TTI::TCK_CodeSize ? 1 : 10;
     SmallVector<unsigned, 2> ISDs;
     switch (IID) {
     default: {
+      // Scalable vectors cannot be scalarized, so return Invalid.
+      if (isa<ScalableVectorType>(RetTy) ||
+          any_of(Tys, [](const Type *Ty) { return isa<ScalableVectorType>(Ty); }))
+        return InstructionCost::getInvalid();
+
       // Assume that we need to scalarize this intrinsic.
       InstructionCost ScalarizationCost = ScalarizationCostPassed;
       unsigned ScalarCalls = 1;
       Type *ScalarRetTy = RetTy;
       if (auto *RetVTy = dyn_cast<VectorType>(RetTy)) {
         if (!SkipScalarizationCost)
           ScalarizationCost = getScalarizationOverhead(RetVTy, true, false);
         ScalarCalls = std::max(ScalarCalls,
if (IID == Intrinsic::experimental_constrained_fmuladd) {
       return thisT()->getIntrinsicInstrCost(FMulAttrs, CostKind) +
              thisT()->getIntrinsicInstrCost(FAddAttrs, CostKind);
     }
 
     // Else, assume that we need to scalarize this intrinsic. For math builtins
     // this will emit a costly libcall, adding call overhead and spills. Make it
     // very expensive.
     if (auto *RetVTy = dyn_cast<VectorType>(RetTy)) {
+      // Scalable vectors cannot be scalarized, so return Invalid.
+      if (isa<ScalableVectorType>(RetTy) ||
+          any_of(Tys, [](const Type *Ty) { return isa<ScalableVectorType>(Ty); }))
+        return InstructionCost::getInvalid();
+
       unsigned ScalarizationCost = SkipScalarizationCost ?
         ScalarizationCostPassed : getScalarizationOverhead(RetVTy, true, false);
 
       unsigned ScalarCalls = cast<FixedVectorType>(RetVTy)->getNumElements();
       SmallVector<Type *, 4> ScalarTys;
       for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) {
         Type *Ty = Tys[i];
         if (Ty->isVectorTy())

llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll

 declare float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float>)
 declare double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double>)
 declare float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float>)
 declare double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double>)
 
 
 define void  @count_zeroes(<vscale x 4 x i32> %A) {
 ; CHECK-LABEL: 'count_zeroes'
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %ctlz = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cttz = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
   %ctlz = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
   %cttz = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
   ret void
 }
 declare <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32>, i1)
 declare <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32>, i1)
 
 

llvm/test/Analysis/CostModel/AArch64/sve-math.ll

 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <vscale x 2 x double> %r
 ;
   %r = fadd <vscale x 2 x double> %a, %b
   ret <vscale x 2 x double> %r
 }
 
 define <vscale x 2 x double> @sqrt_v2f64(<vscale x 2 x double> %a) {
 ; THRU-LABEL: 'sqrt_v2f64'
-; THRU-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> %a)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> %a)
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <vscale x 2 x double> %r
 ;
 ; LATE-LABEL: 'sqrt_v2f64'
 ; LATE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %r = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> %a)
 ; LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <vscale x 2 x double> %r
 ;
 ; SIZE-LABEL: 'sqrt_v2f64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> %a)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> %a)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <vscale x 2 x double> %r
 ;
 ; SIZE_LATE-LABEL: 'sqrt_v2f64'
-; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> %a)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> %a)
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <vscale x 2 x double> %r
 ;
   %r = call <vscale x 2 x double> @llvm.sqrt.v2f64(<vscale x 2 x double> %a)
   ret <vscale x 2 x double> %r
 }