[VPlan] Move VPWidenCallRecipe to VPlanRecipes.cpp (NFC).

[fhahn]

Summary

Depends on D132585.

[Ayal]

Nice cleanup! Minor nits.

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

4155–4156

Remove useless method altogether?

llvm/lib/Transforms/Vectorize/VPlan.h

352 ↗

(On Diff #455318)

typo

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

448

Store ID in call recipe instead of dragging TLI around in State?

[fhahn]

Rebased after improvements to D132585, comments should be addressed now, thanks!

[fhahn]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

4155–4156

Done, it was left over by accident.

llvm/lib/Transforms/Vectorize/VPlan.h

352 ↗

(On Diff #455318)

This is now gone in the latest version.

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

448

Done after the latest update to D132585

[fhahn]

Rebased after recent changes to D132585.

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
     }
   }
 }
 
 bool InnerLoopVectorizer::useOrderedReductions(
     const RecurrenceDescriptor &RdxDesc) {
   return Cost->useOrderedReductions(RdxDesc);
 }
 
-void InnerLoopVectorizer::widenCallInstruction(
-    CallInst &CI, VPValue *Def, VPUser &ArgOperands, VPTransformState &State,
-    Intrinsic::ID VectorIntrinsicID) {
-  assert(!isa<DbgInfoIntrinsic>(CI) &&
-         "DbgInfoIntrinsic should have been dropped during VPlan construction");
-  State.setDebugLocFromInst(&CI);
-
-  SmallVector<Type *, 4> Tys;
-  for (Value *ArgOperand : CI.args())
-    Tys.push_back(ToVectorTy(ArgOperand->getType(), VF.getKnownMinValue()));
-
-  for (unsigned Part = 0; Part < UF; ++Part) {
-    SmallVector<Type *, 2> TysForDecl = {CI.getType()};
-    SmallVector<Value *, 4> Args;
-    for (const auto &I : enumerate(ArgOperands.operands())) {
-      // Some intrinsics have a scalar argument - don't replace it with a
-      // vector.
-      Value *Arg;
-      if (!VectorIntrinsicID ||
-          !isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index()))
-        Arg = State.get(I.value(), Part);
-      else
-        Arg = State.get(I.value(), VPIteration(0, 0));
-      if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index()))
-        TysForDecl.push_back(Arg->getType());
-      Args.push_back(Arg);
-    }
-
-    Function *VectorF;
-    if (VectorIntrinsicID) {
-      // Use vector version of the intrinsic.
-      if (VF.isVector())
-        TysForDecl[0] = VectorType::get(CI.getType()->getScalarType(), VF);
-      Module *M = State.Builder.GetInsertBlock()->getModule();
-      VectorF = Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl);
-      assert(VectorF && "Can't retrieve vector intrinsic.");
-    } else {
-      // Use vector version of the function call.
-      const VFShape Shape = VFShape::get(CI, VF, false /*HasGlobalPred*/);
-#ifndef NDEBUG
-      assert(VFDatabase(CI).getVectorizedFunction(Shape) != nullptr &&
-             "Can't create vector function.");
-#endif
-      VectorF = VFDatabase(CI).getVectorizedFunction(Shape);
-    }
-      SmallVector<OperandBundleDef, 1> OpBundles;
-      CI.getOperandBundlesAsDefs(OpBundles);
-      CallInst *V = Builder.CreateCall(VectorF, Args, OpBundles);
-
-      if (isa<FPMathOperator>(V))
-        V->copyFastMathFlags(&CI);
-
-      State.set(Def, V, Part);
-      State.addMetadata(V, &CI);
-  }
-}
-
 void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
   // We should not collect Scalars more than once per VF. Right now, this
   // function is called from collectUniformsAndScalars(), which already does
   // this check. Collecting Scalars for VF=1 does not make any sense.
   assert(VF.isVector() && Scalars.find(VF) == Scalars.end() &&
          "This function should not be visited twice for the same VF");
 
   // This avoids any chances of creating a REPLICATE recipe during planning
   // since that would result in generation of scalarized code during execution,
if (getNumStoreOperands() > 0) {
       getVPValue(OpIdx)->printAsOperand(O, SlotTracker);
       O << " = load from index " << i;
     }
     ++OpIdx;
   }
 }
 #endif
 
-void VPWidenCallRecipe::execute(VPTransformState &State) {
-  State.ILV->widenCallInstruction(*cast<CallInst>(getUnderlyingInstr()), this,
-                                  *this, State, VectorIntrinsicID);
-}
-
 void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
   assert(!State.Instance && "Int or FP induction being replicated.");
 
   Value *Start = getStartValue()->getLiveInIRValue();
   const InductionDescriptor &ID = getInductionDescriptor();
   TruncInst *Trunc = getTruncInst();
   IRBuilderBase &Builder = State.Builder;
   assert(IV->getType() == ID.getStartValue()->getType() && "Types must match");

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

void VPInstruction::setFastMathFlags(FastMathFlags FMFNew) {
   assert((Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
           Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
           Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
           Opcode == Instruction::FCmp) &&
          "this op can't take fast-math flags");
   FMF = FMFNew;
 }
 
+void VPWidenCallRecipe::execute(VPTransformState &State) {
+  auto &CI = *cast<CallInst>(getUnderlyingInstr());
+  assert(!isa<DbgInfoIntrinsic>(CI) &&
+         "DbgInfoIntrinsic should have been dropped during VPlan construction");
+  State.setDebugLocFromInst(&CI);
+
+  SmallVector<Type *, 4> Tys;
+  for (Value *ArgOperand : CI.args())
+    Tys.push_back(
+        ToVectorTy(ArgOperand->getType(), State.VF.getKnownMinValue()));
+
+  for (unsigned Part = 0; Part < State.UF; ++Part) {
+    SmallVector<Type *, 2> TysForDecl = {CI.getType()};
+    SmallVector<Value *, 4> Args;
+    for (const auto &I : enumerate(operands())) {
+      // Some intrinsics have a scalar argument - don't replace it with a
+      // vector.
+      Value *Arg;
+      if (VectorIntrinsicID == Intrinsic::not_intrinsic ||
+          !isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index()))
+        Arg = State.get(I.value(), Part);
+      else
+        Arg = State.get(I.value(), VPIteration(0, 0));
+      if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index()))
+        TysForDecl.push_back(Arg->getType());
+      Args.push_back(Arg);
+    }
+
+    Function *VectorF;
+    if (VectorIntrinsicID != Intrinsic::not_intrinsic) {
+      // Use vector version of the intrinsic.
+      if (State.VF.isVector())
+        TysForDecl[0] =
+            VectorType::get(CI.getType()->getScalarType(), State.VF);
+      Module *M = State.Builder.GetInsertBlock()->getModule();
+      VectorF = Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl);
+      assert(VectorF && "Can't retrieve vector intrinsic.");
+    } else {
+      // Use vector version of the function call.
+      const VFShape Shape = VFShape::get(CI, State.VF, false /*HasGlobalPred*/);
+#ifndef NDEBUG
+      assert(VFDatabase(CI).getVectorizedFunction(Shape) != nullptr &&
+             "Can't create vector function.");
+#endif
+      VectorF = VFDatabase(CI).getVectorizedFunction(Shape);
+    }
+    SmallVector<OperandBundleDef, 1> OpBundles;
+    CI.getOperandBundlesAsDefs(OpBundles);
+    CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
+
+    if (isa<FPMathOperator>(V))
+      V->copyFastMathFlags(&CI);
+
+    State.set(this, V, Part);
+    State.addMetadata(V, &CI);
+  }
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent,
                               VPSlotTracker &SlotTracker) const {
   O << Indent << "WIDEN-CALL ";
 
   auto *CI = cast<CallInst>(getUnderlyingInstr());
   if (CI->getType()->isVoidTy())
     O << "void ";