[AArch64][SLP] Adjust cost estimation for scalar FMUL+FADD/FSUB to be with possibility of using scalar FMA instead.

[fhahn]

FYI there are a few different patches floating around to improve the situation: D132872 and linked ones. Any chance those help with the cases you are looking at?

llvm/include/llvm/Analysis/TargetTransformInfo.h

struct VPLegalization {
 
     bool shouldDoNothing() const {
       return (EVLParamStrategy == Legal) && (OpStrategy == Legal);
     }
     VPLegalization(VPTransform EVLParamStrategy, VPTransform OpStrategy)
         : EVLParamStrategy(EVLParamStrategy), OpStrategy(OpStrategy) {}
   };
 
+  bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const;
+
   /// \returns How the target needs this vector-predicated operation to be
   /// transformed.
   VPLegalization getVPLegalizationStrategy(const VPIntrinsic &PI) const;
   /// @}
 
   /// @}
 
 private:
public:
 
   virtual bool shouldExpandReduction(const IntrinsicInst *II) const = 0;
   virtual unsigned getGISelRematGlobalCost() const = 0;
   virtual unsigned getMinTripCountTailFoldingThreshold() const = 0;
   virtual bool enableScalableVectorization() const = 0;
   virtual bool supportsScalableVectors() const = 0;
   virtual bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
                                      Align Alignment) const = 0;
+  virtual bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const = 0;
   virtual VPLegalization
   getVPLegalizationStrategy(const VPIntrinsic &PI) const = 0;
 };
 
 template <typename T>
 class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   T Impl;
 
bool enableScalableVectorization() const override {
     return Impl.enableScalableVectorization();
   }
 
   bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
                              Align Alignment) const override {
     return Impl.hasActiveVectorLength(Opcode, DataType, Alignment);
   }
 
+  bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const override {
+    return Impl.isFMAFasterThanFMulAndFAdd(F, Ty);
+  }
+
   VPLegalization
   getVPLegalizationStrategy(const VPIntrinsic &PI) const override {
     return Impl.getVPLegalizationStrategy(PI);
   }
 };
 
 template <typename T>
 TargetTransformInfo::TargetTransformInfo(T Impl)

llvm/include/llvm/Analysis/TargetTransformInfoImpl.h

public:
 
   bool enableScalableVectorization() const { return false; }
 
   bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
                              Align Alignment) const {
     return false;
   }
 
+  bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const {
+    return false;
+  }
+
   TargetTransformInfo::VPLegalization
   getVPLegalizationStrategy(const VPIntrinsic &PI) const {
     return TargetTransformInfo::VPLegalization(
         /* EVLParamStrategy */ TargetTransformInfo::VPLegalization::Discard,
         /* OperatorStrategy */ TargetTransformInfo::VPLegalization::Convert);
   }
 
 protected:

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h

public:
   bool isLegalToVectorizeReduction(const RecurrenceDescriptor &RdxDesc,
                                    ElementCount VF) const;
 
   bool preferPredicatedReductionSelect(unsigned Opcode, Type *Ty,
                                        TTI::ReductionFlags Flags) const {
     return ST->hasSVE();
   }
 
+  bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const {
+    return TLI->isFMAFasterThanFMulAndFAdd(F, Ty);
+  }
+
   InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
                                              Optional<FastMathFlags> FMF,
                                              TTI::TargetCostKind CostKind);
 
   InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
                                  ArrayRef<int> Mask,
                                  TTI::TargetCostKind CostKind, int Index,
                                  VectorType *SubTp,

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

if (Last->State != TreeEntry::NeedToGather) {
       // Update the scheduler bundle to point to this TreeEntry.
       ScheduleData *BundleMember = *Bundle;
       assert((BundleMember || isa<PHINode>(S.MainOp) ||
               isVectorLikeInstWithConstOps(S.MainOp) ||
               doesNotNeedToSchedule(VL)) &&
              "Bundle and VL out of sync");
       if (BundleMember) {
         for (Value *V : VL) {
+          if (isa<Instruction>(V)) {
+            Instruction *Op = dyn_cast<Instruction>(V);
+            if ((Op->getOpcode() == Instruction::FAdd ||
+                 Op->getOpcode() == Instruction::FSub) && Op->hasOneUse()) {
+              if (isa<Instruction>(V))
+                for (int i = 0; i<2; ++i) {
+                  Value *VOp = Op->getOperand(i);
+                  Instruction *Op1 = dyn_cast<Instruction>(VOp);
+                  if (Op1 && Op1->getOpcode() == Instruction::FMul && Op1->hasOneUse()) {
+                    LLVM_DEBUG(dbgs() << "MyDebug0: " << *Op << " Op1" << *Op1 << "\n");
+                    break;
+                  }
+                }
+            }
+          }
           if (doesNotNeedToBeScheduled(V))
             continue;
           assert(BundleMember && "Unexpected end of bundle.");
           BundleMember->TE = Last;
           BundleMember = BundleMember->NextInBundle;
         }
       }
       assert(!BundleMember && "Bundle and VL out of sync");
InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
   case Instruction::And:
   case Instruction::Or:
   case Instruction::Xor:
   case Instruction::GetElementPtr: {
     unsigned Opcode = ShuffleOrOp == Instruction::GetElementPtr
                           ? static_cast<unsigned>(Instruction::Add)
                           : ShuffleOrOp;
     auto GetScalarCost = [=](unsigned Idx) {
+      //LLVM_DEBUG(dbgs() << "MyDebug0: " << VL.size() << " Idx: " << Idx << "\n");
+      //for (Value *V : VL)
+      //  LLVM_DEBUG(dbgs() << "MyDebug0: " << *V << "\n");
       auto *VI = dyn_cast<Instruction>(VL[Idx]);
       // GEPs may contain just addresses without instructions, consider
       // their cost 0.
       if (!VI)
         return InstructionCost();
       unsigned OpIdx = isa<UnaryOperator>(VI) ? 0 : 1;
       TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(VI->getOperand(0));
       TTI::OperandValueInfo Op2Info =
           TTI::getOperandInfo(VI->getOperand(OpIdx));
       SmallVector<const Value *> Operands(VI->operand_values());
+      if ((Opcode == Instruction::FAdd || Opcode == Instruction::FSub) && VI->hasOneUse()) {
+        bool HasFMul = false;
+        //LLVM_DEBUG(dbgs() << "MyDebug1: " << *VI << "\n");
+        for (int i = 0; i<2; ++i) {
+          Value *V = VI->getOperand(i);
+          if (isa<Instruction>(V)) {
+             Instruction *Op = dyn_cast<Instruction>(V);
+             if (Op->getOpcode() == Instruction::FMul && Op->hasOneUse()) {
+               HasFMul = true;
+               break;
+             }
+          }
+        }
+        if (HasFMul) {
+          LLVM_DEBUG(dbgs() << "MyDebug2: " << *VI << "\n");
+          InstructionCost C = 1;
+          //assert(false);
+          return C;
+        }
+      }
       return TTI->getArithmeticInstrCost(Opcode, ScalarTy, CostKind, Op1Info,
                                          Op2Info, Operands, VI);
     };
     auto GetVectorCost = [=](InstructionCost CommonCost) {
       unsigned OpIdx = isa<UnaryOperator>(VL0) ? 0 : 1;
       TTI::OperandValueInfo Op1Info = getOperandInfo(VL, 0);
       TTI::OperandValueInfo Op2Info = getOperandInfo(VL, OpIdx);
       return TTI->getArithmeticInstrCost(Opcode, VecTy, CostKind, Op1Info,
auto TryFindNodeWithEqualOperands = [=]() {
               TE->getAltOpcode() == E->getAltOpcode()) ||
              (TE->getOpcode() == E->getAltOpcode() &&
               TE->getAltOpcode() == E->getOpcode())) &&
             TE->hasEqualOperands(*E))
           return true;
       }
       return false;
     };
-    auto GetScalarCost = [=](unsigned Idx) {
+    //const Function *F1 = F;
+    auto GetScalarCost = [=](unsigned Idx/*, const Function *F1*/) {
       auto *VI = cast<Instruction>(VL[Idx]);
+      InstructionCost C = 1;
       assert(E->isOpcodeOrAlt(VI) && "Unexpected main/alternate opcode");
       (void)E;
+      unsigned Opcode = VI->getOpcode();
+      if ((Opcode == Instruction::FAdd || Opcode == Instruction::FSub) && VI->hasOneUse()) {
+        bool HasFMul = false;
+        //LLVM_DEBUG(dbgs() << "MyDebug1: " << *VI << "\n");
+        for (int i = 0; i<2; ++i) {
+          Value *V = VI->getOperand(i);
+          if (isa<Instruction>(V)) {
+            Instruction *Op = dyn_cast<Instruction>(V);
+            if (Op->getOpcode() == Instruction::FMul && Op->hasOneUse()) {
+              HasFMul = true;
+              break;
+            }
+          }
+        }
+        if (HasFMul) {
+          LLVM_DEBUG(dbgs() << "MyDebug2: " << *VI << "\n");
+          LLVM_DEBUG(dbgs() << "MyDebug3: From: " << TTI->getInstructionCost(VI, CostKind)
+                     << " To: " << C << "\n");
+          //TTI->isFMAFasterThanFMulAndFAdd(&(*F1), VI->getType());
+          //assert(false);
+          return C;
+        }
+      }
       return TTI->getInstructionCost(VI, CostKind);
     };
     // Need to clear CommonCost since the final shuffle cost is included into
     // vector cost.
     auto GetVectorCost = [&](InstructionCost) {
       // VecCost is equal to sum of the cost of creating 2 vectors
       // and the cost of creating shuffle.
       InstructionCost VecCost = 0;