[Legalizer][ARM][AArch64] Add legalizations for VECREDUCE_SEQ_FMUL

[cameron.mcinally]

Summary

Hook up legalizations for VECREDUCE_SEQ_FMUL. This is following up on the VECREDUCE_SEQ_FADD work from D90247.

[cameron.mcinally]

Reformat to appease pre-merge checks...

[nikic]

LGTM

Before you land this, I would suggest to improve the test coverage a bit:

• In @llvm.vector.reduce.fmul.f128.v2f128, duplicate the tests without fast
• In llvm/test/CodeGen/ARM/vecreduce-fmul-legalization-strict.ll and llvm/test/CodeGen/AArch64/vecreduce-fmul-legalization-strict.ll, use 1.0 instead of 0.0 as the start value. That was probably a copy&paste mistake from fadds.

[cameron.mcinally]

In D90644#2372150, @nikic wrote:

• In llvm/test/CodeGen/ARM/vecreduce-fmul-legalization-strict.ll and llvm/test/CodeGen/AArch64/vecreduce-fmul-legalization-strict.ll, use 1.0 instead of 0.0 as the start value. That was probably a copy&paste mistake from fadds.

That caught my eye too, but the 0.0 seemed okay since we can't peep this without NSZ (-0*0) and NNAN (0*NaN). Changing it to 1.0 isn't a big deal though...

[cameron.mcinally]

In D90644#2372260, @cameron.mcinally wrote:

In D90644#2372150, @nikic wrote:

• In llvm/test/CodeGen/ARM/vecreduce-fmul-legalization-strict.ll and llvm/test/CodeGen/AArch64/vecreduce-fmul-legalization-strict.ll, use 1.0 instead of 0.0 as the start value. That was probably a copy&paste mistake from fadds.

That caught my eye too, but the 0.0 seemed okay since we can't peep this without NSZ (-0*0) and NNAN (0*NaN). Changing it to 1.0 isn't a big deal though...

Oh, I see it now. The 1.0 is the neutral element, so it folds. Makes sense now...

llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp

#endif
     case ISD::VAARG:       R = SoftenFloatRes_VAARG(N); break;
     case ISD::VECREDUCE_FADD:
     case ISD::VECREDUCE_FMUL:
     case ISD::VECREDUCE_FMIN:
     case ISD::VECREDUCE_FMAX:
       R = SoftenFloatRes_VECREDUCE(N);
       break;
     case ISD::VECREDUCE_SEQ_FADD:
+    case ISD::VECREDUCE_SEQ_FMUL:
       R = SoftenFloatRes_VECREDUCE_SEQ(N);
       break;
   }
 
   // If R is null, the sub-method took care of registering the result.
   if (R.getNode()) {
     assert(R.getNode() != N);
     SetSoftenedFloat(SDValue(N, ResNo), R);
#endif
     case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
     case ISD::VECREDUCE_FADD:
     case ISD::VECREDUCE_FMUL:
     case ISD::VECREDUCE_FMIN:
     case ISD::VECREDUCE_FMAX:
       R = PromoteFloatRes_VECREDUCE(N);
       break;
     case ISD::VECREDUCE_SEQ_FADD:
+    case ISD::VECREDUCE_SEQ_FMUL:
       R = PromoteFloatRes_VECREDUCE_SEQ(N);
       break;
   }
 
   if (R.getNode())
     SetPromotedFloat(SDValue(N, ResNo), R);
 }
 
#endif
   case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
   case ISD::VECREDUCE_FADD:
   case ISD::VECREDUCE_FMUL:
   case ISD::VECREDUCE_FMIN:
   case ISD::VECREDUCE_FMAX:
     R = SoftPromoteHalfRes_VECREDUCE(N);
     break;
   case ISD::VECREDUCE_SEQ_FADD:
+  case ISD::VECREDUCE_SEQ_FMUL:
     R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
     break;
   }
 
   if (R.getNode())
     SetSoftPromotedHalf(SDValue(N, ResNo), R);
 }
 

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp

#include "llvm/IR/ConstrainedOps.def"
   case ISD::VECREDUCE_FADD:
   case ISD::VECREDUCE_FMUL:
   case ISD::VECREDUCE_FMAX:
   case ISD::VECREDUCE_FMIN:
     Action = TLI.getOperationAction(Node->getOpcode(),
                                     Node->getOperand(0).getValueType());
     break;
   case ISD::VECREDUCE_SEQ_FADD:
+  case ISD::VECREDUCE_SEQ_FMUL:
     Action = TLI.getOperationAction(Node->getOpcode(),
                                     Node->getOperand(1).getValueType());
     break;
   }
 
   LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
 
   SmallVector<SDValue, 8> ResultVals;
#include "llvm/IR/ConstrainedOps.def"
   case ISD::VECREDUCE_UMIN:
   case ISD::VECREDUCE_FADD:
   case ISD::VECREDUCE_FMUL:
   case ISD::VECREDUCE_FMAX:
   case ISD::VECREDUCE_FMIN:
     Results.push_back(TLI.expandVecReduce(Node, DAG));
     return;
   case ISD::VECREDUCE_SEQ_FADD:
+  case ISD::VECREDUCE_SEQ_FMUL:
     Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
     return;
   case ISD::SREM:
   case ISD::UREM:
     ExpandREM(Node, Results);
     return;
   }
 

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

#endif
     case ISD::VECREDUCE_SMIN:
     case ISD::VECREDUCE_UMAX:
     case ISD::VECREDUCE_UMIN:
     case ISD::VECREDUCE_FMAX:
     case ISD::VECREDUCE_FMIN:
       Res = ScalarizeVecOp_VECREDUCE(N);
       break;
     case ISD::VECREDUCE_SEQ_FADD:
+    case ISD::VECREDUCE_SEQ_FMUL:
       Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
       break;
     }
   }
 
   // If the result is null, the sub-method took care of registering results etc.
   if (!Res.getNode()) return false;
 
#endif
     case ISD::VECREDUCE_SMIN:
     case ISD::VECREDUCE_UMAX:
     case ISD::VECREDUCE_UMIN:
     case ISD::VECREDUCE_FMAX:
     case ISD::VECREDUCE_FMIN:
       Res = SplitVecOp_VECREDUCE(N, OpNo);
       break;
     case ISD::VECREDUCE_SEQ_FADD:
+    case ISD::VECREDUCE_SEQ_FMUL:
       Res = SplitVecOp_VECREDUCE_SEQ(N);
       break;
     }
   }
 
   // If the result is null, the sub-method took care of registering results etc.
   if (!Res.getNode()) return false;
 
#endif
   case ISD::VECREDUCE_SMIN:
   case ISD::VECREDUCE_UMAX:
   case ISD::VECREDUCE_UMIN:
   case ISD::VECREDUCE_FMAX:
   case ISD::VECREDUCE_FMIN:
     Res = WidenVecOp_VECREDUCE(N);
     break;
   case ISD::VECREDUCE_SEQ_FADD:
+  case ISD::VECREDUCE_SEQ_FMUL:
     Res = WidenVecOp_VECREDUCE_SEQ(N);
     break;
   }
 
   // If Res is null, the sub-method took care of registering the result.
   if (!Res.getNode()) return false;
 
   // If the result is N, the sub-method updated N in place.  Tell the legalizer

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

 ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
   switch (VecReduceOpcode) {
   default:
     llvm_unreachable("Expected VECREDUCE opcode");
   case ISD::VECREDUCE_FADD:
   case ISD::VECREDUCE_SEQ_FADD:
     return ISD::FADD;
   case ISD::VECREDUCE_FMUL:
+  case ISD::VECREDUCE_SEQ_FMUL:
     return ISD::FMUL;
   case ISD::VECREDUCE_ADD:
     return ISD::ADD;
   case ISD::VECREDUCE_MUL:
     return ISD::MUL;
   case ISD::VECREDUCE_AND:
     return ISD::AND;
   case ISD::VECREDUCE_OR:

llvm/lib/CodeGen/TargetLoweringBase.cpp

#include "llvm/IR/ConstrainedOps.def"
     setOperationAction(ISD::VECREDUCE_XOR, VT, Expand);
     setOperationAction(ISD::VECREDUCE_SMAX, VT, Expand);
     setOperationAction(ISD::VECREDUCE_SMIN, VT, Expand);
     setOperationAction(ISD::VECREDUCE_UMAX, VT, Expand);
     setOperationAction(ISD::VECREDUCE_UMIN, VT, Expand);
     setOperationAction(ISD::VECREDUCE_FMAX, VT, Expand);
     setOperationAction(ISD::VECREDUCE_FMIN, VT, Expand);
     setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Expand);
+    setOperationAction(ISD::VECREDUCE_SEQ_FMUL, VT, Expand);
   }
 
   // Most targets ignore the @llvm.prefetch intrinsic.
   setOperationAction(ISD::PREFETCH, MVT::Other, Expand);
 
   // Most targets also ignore the @llvm.readcyclecounter intrinsic.
   setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Expand);
 

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h

int getInterleavedMemoryOpCost(
       Align Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
       bool UseMaskForCond = false, bool UseMaskForGaps = false);
 
   bool
   shouldConsiderAddressTypePromotion(const Instruction &I,
                                      bool &AllowPromotionWithoutCommonHeader);
 
   bool shouldExpandReduction(const IntrinsicInst *II) const {
-    switch (II->getIntrinsicID()) {
-    case Intrinsic::vector_reduce_fmul:
-      // We don't have legalization support for ordered FMUL reductions.
-      return !II->getFastMathFlags().allowReassoc();
-
-    default:
-      // Don't expand anything else, let legalization deal with it.
-      return false;
-    }
-  }
 
   unsigned getGISelRematGlobalCost() const {
     return 2;
   }
 
   bool useReductionIntrinsic(unsigned Opcode, Type *Ty,
                              TTI::ReductionFlags Flags) const;
 

llvm/lib/Target/ARM/ARMTargetTransformInfo.h

bool useReductionIntrinsic(unsigned Opcode, Type *Ty,
                              TTI::ReductionFlags Flags) const;
 
   bool preferInLoopReduction(unsigned Opcode, Type *Ty,
                              TTI::ReductionFlags Flags) const;
 
   bool preferPredicatedReductionSelect(unsigned Opcode, Type *Ty,
                                        TTI::ReductionFlags Flags) const;
 
   bool shouldExpandReduction(const IntrinsicInst *II) const {
-    switch (II->getIntrinsicID()) {
-    case Intrinsic::vector_reduce_fmul:
-      // We don't have legalization support for ordered FMUL reductions.
-      return !II->getFastMathFlags().allowReassoc();
-    default:
-      // Don't expand anything else, let legalization deal with it.
-      return false;
-    }
-  }
 
   int getCFInstrCost(unsigned Opcode,
                      TTI::TargetCostKind CostKind);
 
   int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                        TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
                        const Instruction *I = nullptr);