[SVE][LoopVectorize] Support for vectorization of loops with function calls

[kmclaughlin]

Summary

Changes getScalarizationOverhead to return an invalid cost for scalable VFs
and adds some simple tests for loops containing a function for which
there is a vectorized variant available.

[david-arm]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

4968–4969

nit: I think we can remove the braces now.

llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll

3

Is it worth adding tests for LLVM intrinsics too that use fast math, i.e. test/Transforms/LoopVectorize/PowerPC/widened-massv-vfabi-attr.ll has an example like this:

  %1 = call fast double @llvm.sin.f64(double %conv) #1
  %add = fadd fast double %Sum.0, %1
...
declare double @llvm.sin.f64(double) #0
declare <2 x double> @__sind2_massv(<2 x double>) #0
attributes #0 = { nounwind readnone speculatable willreturn }
attributes #1 = { "vector-function-abi-variant"="_ZGV_LLVM_N2v_llvm.sin.f64(__sind2_massv)" }

[kmclaughlin]

• Removed unnecessary braces from widenCallInstruction
• Added a test for a loop containing an LLVM intrinsic (@llvm.sin.f64)

[david-arm]

LGTM! Thanks for the new test!

[kmclaughlin]

• Added the -force-vector-interleave=1 flag to the scalable-call.ll

[david-arm]

llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll

78

nit: If you could make the tests consistent in terms of the numbers of CHECK lines before merging that would be great! Perhaps disable interleaving with -force-vector-interleave=1?

llvm/lib/Analysis/TargetTransformInfo.cpp

IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id,
   ParamTys.insert(ParamTys.begin(), FTy->param_begin(), FTy->param_end());
 }
 
 IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id,
                                                  const CallBase &CI,
                                                  ElementCount Factor)
     : RetTy(CI.getType()), IID(Id), VF(Factor) {
 
-  assert(!Factor.isScalable() && "Scalable vectors are not yet supported");
   if (auto *FPMO = dyn_cast<FPMathOperator>(&CI))
     FMF = FPMO->getFastMathFlags();
 
   Arguments.insert(Arguments.begin(), CI.arg_begin(), CI.arg_end());
   FunctionType *FTy =
     CI.getCalledFunction()->getFunctionType();
   ParamTys.insert(ParamTys.begin(), FTy->param_begin(), FTy->param_end());
 }

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) {
 
     CSEMap[In] = In;
   }
 }
 
 InstructionCost
 LoopVectorizationCostModel::getVectorCallCost(CallInst *CI, ElementCount VF,
                                               bool &NeedToScalarize) {
-  assert(!VF.isScalable() && "scalable vectors not yet supported.");
   Function *F = CI->getCalledFunction();
   Type *ScalarRetTy = CI->getType();
   SmallVector<Type *, 4> Tys, ScalarTys;
   for (auto &ArgOp : CI->arg_operands())
     ScalarTys.push_back(ArgOp->getType());
 
   // Estimate cost of scalarized vector call. The source operands are assumed
   // to be vectors, so we need to extract individual elements from there,
for (auto &I : enumerate(ArgOperands.operands())) {
       else
         Arg = State.get(I.value(), VPIteration(0, 0));
       Args.push_back(Arg);
     }
 
     Function *VectorF;
     if (UseVectorIntrinsic) {
       // Use vector version of the intrinsic.
       Type *TysForDecl[] = {CI->getType()};
-      if (VF.isVector()) {
-        assert(!VF.isScalable() && "VF is assumed to be non scalable.");
+      if (VF.isVector())
         TysForDecl[0] = VectorType::get(CI->getType()->getScalarType(), VF);
-      }
       VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl);
       assert(VectorF && "Can't retrieve vector intrinsic.");
     } else {
       // Use vector version of the function call.
       const VFShape Shape = VFShape::get(*CI, VF, false /*HasGlobalPred*/);
 #ifndef NDEBUG
       assert(VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr &&
              "Can't create vector function.");
bool TypeNotScalarized =
       TTI.getNumberOfParts(VectorTy) < VF.getKnownMinValue();
   return VectorizationCostTy(C, TypeNotScalarized);
 }
 
 InstructionCost
 LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
                                                      ElementCount VF) {
 
-  assert(!VF.isScalable() &&
-         "cannot compute scalarization overhead for scalable vectorization");
+  if (VF.isScalable())
+    return InstructionCost::getInvalid();
+
   if (VF.isScalar())
     return 0;
 
   InstructionCost Cost = 0;
   Type *RetTy = ToVectorTy(I->getType(), VF);
   if (!RetTy->isVoidTy() &&
       (!isa<LoadInst>(I) || !TTI.supportsEfficientVectorElementLoadStore()))
     Cost += TTI.getScalarizationOverhead(

llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll

+; RUN: opt -S -loop-vectorize -force-vector-interleave=1 -instcombine -mattr=+sve -mtriple aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @vec_load(i64 %N, double* nocapture %a, double* nocapture readonly %b) {
+; CHECK-LABEL: @vec_load
+; CHECK: vector.body:
+; CHECK: %[[LOAD:.*]] = load <vscale x 2 x double>, <vscale x 2 x double>*
+; CHECK: call <vscale x 2 x double> @foo_vec(<vscale x 2 x double> %[[LOAD]])
+entry:
+  %cmp7 = icmp sgt i64 %N, 0
+  br i1 %cmp7, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %b, i64 %iv
+  %0 = load double, double* %arrayidx, align 8
+  %1 = call double @foo(double %0) #0
+  %add = fadd double %1, 1.000000e+00
+  %arrayidx2 = getelementptr inbounds double, double* %a, i64 %iv
+  store double %add, double* %arrayidx2, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !1
+
+for.end:                                 ; preds = %for.body, %entry
+  ret void
+}
+
+define void @vec_scalar(i64 %N, double* nocapture %a) {
+; CHECK-LABEL: @vec_scalar
+; CHECK: vector.body:
+; CHECK: call <vscale x 2 x double> @foo_vec(<vscale x 2 x double> shufflevector (<vscale x 2 x double> insertelement (<vscale x 2 x double> poison, double 1.000000e+01, i32 0), <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer))
+entry:
+  %cmp7 = icmp sgt i64 %N, 0
+  br i1 %cmp7, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %0 = call double @foo(double 10.0) #0
+  %sub = fsub double %0, 1.000000e+00
+  %arrayidx = getelementptr inbounds double, double* %a, i64 %iv
+  store double %sub, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !1
+
+for.end:                                 ; preds = %for.body, %entry
+  ret void
+}
+
+define void @vec_ptr(i64 %N, i64* noalias %a, i64** readnone %b) {
+; CHECK-LABEL: @vec_ptr
+; CHECK: vector.body:
+; CHECK: %[[LOAD:.*]] = load <vscale x 2 x i64*>, <vscale x 2 x i64*>*
+; CHECK: call <vscale x 2 x i64> @bar_vec(<vscale x 2 x i64*> %[[LOAD]])
+entry:
+  %cmp7 = icmp sgt i64 %N, 0
+  br i1 %cmp7, label %for.body, label %for.end
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep = getelementptr i64*, i64** %b, i64 %iv
+  %load = load i64*, i64** %gep
+  %call = call i64 @bar(i64* %load) #1
+  %arrayidx = getelementptr inbounds i64, i64* %a, i64 %iv
+  store i64 %call, i64* %arrayidx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1
+
+for.end:
+  ret void
+}
+
+define void @vec_intrinsic(i64 %N, double* nocapture readonly %a) {
+; CHECK-LABEL: @vec_intrinsic
+; CHECK: vector.body:
+; CHECK: %[[LOAD:.*]] = load <vscale x 2 x double>, <vscale x 2 x double>*
+; CHECK: call fast <vscale x 2 x double> @sin_vec(<vscale x 2 x double> %[[LOAD]])
+entry:
+  %cmp7 = icmp sgt i64 %N, 0
+  br i1 %cmp7, label %for.body, label %for.end
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %a, i64 %iv
+  %0 = load double, double* %arrayidx, align 8
+  %1 = call fast double @llvm.sin.f64(double %0) #2
+  %add = fadd fast double %1, 1.000000e+00
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1
+
+for.end:
+  ret void
+}
+
+declare double @foo(double)
+declare i64 @bar(i64*)
+declare double @llvm.sin.f64(double)
+
+declare <vscale x 2 x double> @foo_vec(<vscale x 2 x double>)
+declare <vscale x 2 x i64> @bar_vec(<vscale x 2 x i64*>)
+declare <vscale x 2 x double> @sin_vec(<vscale x 2 x double>)
+
+attributes #0 = { "vector-function-abi-variant"="_ZGV_LLVM_Nxv_foo(foo_vec)" }
+attributes #1 = { "vector-function-abi-variant"="_ZGV_LLVM_Nxv_bar(bar_vec)" }
+attributes #2 = { "vector-function-abi-variant"="_ZGV_LLVM_Nxv_llvm.sin.f64(sin_vec)" }
+
+!1 = distinct !{!1, !2, !3}
+!2 = !{!"llvm.loop.vectorize.width", i32 2}
+!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}