[Legalize][ARM][X86] Add float legalization for VECREDUCE

[nikic]

Summary

This adds SoftenFloatRes, PromoteFloatRes and SoftPromoteHalfRes legalizations for VECREDUCE, to fill the remaining hole in the SDAG legalization. These legalizations simplify expand the reduction and let it be recursively legalized. For the PromoteFloatRes case at least it is possible to do better than that, but it's pretty tricky (because we need to consider the interaction of three different vector legalizations and the type promotion) and probably not really worthwhile.

I haven't added ExpandFloatRes support, as I am not familiar with ppc_fp128.

[nikic]

Add missing cases

[nikic]

Add SoftPromoteHalfRes as well. As we do expand non-fast reductions on X86, we can run into it there. The added tests were previously asserting.

[dmgreen]

This looks good to me. It may be worth adding some fmin/fmax tests with the fadds, just because they can end up being a little different.

llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp

#endif
     case ISD::SELECT_CC:   R = SoftenFloatRes_SELECT_CC(N); break;
     case ISD::FREEZE:      R = SoftenFloatRes_FREEZE(N); break;
     case ISD::STRICT_SINT_TO_FP:
     case ISD::STRICT_UINT_TO_FP:
     case ISD::SINT_TO_FP:
     case ISD::UINT_TO_FP:  R = SoftenFloatRes_XINT_TO_FP(N); break;
     case ISD::UNDEF:       R = SoftenFloatRes_UNDEF(N); break;
     case ISD::VAARG:       R = SoftenFloatRes_VAARG(N); break;
+    case ISD::VECREDUCE_FADD:
+    case ISD::VECREDUCE_FMUL:
+    case ISD::VECREDUCE_FMIN:
+    case ISD::VECREDUCE_FMAX:
+      R = SoftenFloatRes_VECREDUCE(N);
+      break;
   }
 
   // If R is null, the sub-method took care of registering the result.
   if (R.getNode()) {
     assert(R.getNode() != N);
     SetSoftenedFloat(SDValue(N, ResNo), R);
   }
 }
std::pair<SDValue, SDValue> Tmp =
       TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
                       Op, CallOptions, dl, Chain);
 
   if (IsStrict)
     ReplaceValueWith(SDValue(N, 1), Tmp.second);
   return Tmp.first;
 }
 
+SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
+  // Expand and soften recursively.
+  ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
+  return SDValue();
+}
+
 
 //===----------------------------------------------------------------------===//
 //  Convert Float Operand to Integer
 //===----------------------------------------------------------------------===//
 
 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
   LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG);
              dbgs() << "\n");
   SDValue Res = SDValue();
#endif
     case ISD::LOAD:       R = PromoteFloatRes_LOAD(N); break;
     case ISD::SELECT:     R = PromoteFloatRes_SELECT(N); break;
     case ISD::SELECT_CC:  R = PromoteFloatRes_SELECT_CC(N); break;
 
     case ISD::SINT_TO_FP:
     case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
     case ISD::UNDEF:      R = PromoteFloatRes_UNDEF(N); break;
     case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
+    case ISD::VECREDUCE_FADD:
+    case ISD::VECREDUCE_FMUL:
+      R = PromoteFloatRes_VECREDUCE(N);
+      break;
   }
 
   if (R.getNode())
     SetPromotedFloat(SDValue(N, ResNo), R);
 }
 
 // Bitcast from i16 to f16:  convert the i16 to a f32 value instead.
 // At this point, it is not possible to determine if the bitcast value is
return DAG.getNode(
       DAG.getNode(ISD::FP_ROUND, DL, VT, NV, DAG.getIntPtrConstant(0, DL)));
 }
 
 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
   return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
                                                N->getValueType(0)));
 }
 
+SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
+  // Expand and promote recursively.
+  // TODO: This is non-optimal, but dealing with the concurrently happening
+  // vector-legalization is non-trivial. We could do something similar to
+  // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
+  ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
+  return SDValue();
+}
+
 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
   EVT VT = N->getValueType(0);
 
   AtomicSDNode *AM = cast<AtomicSDNode>(N);
   SDLoc SL(N);
 
   SDValue CastVal = BitConvertToInteger(AM->getVal());
   EVT CastVT = CastVal.getValueType();

llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

private:
   SDValue SoftenFloatRes_FSUB(SDNode *N);
   SDValue SoftenFloatRes_FTRUNC(SDNode *N);
   SDValue SoftenFloatRes_LOAD(SDNode *N);
   SDValue SoftenFloatRes_SELECT(SDNode *N);
   SDValue SoftenFloatRes_SELECT_CC(SDNode *N);
   SDValue SoftenFloatRes_UNDEF(SDNode *N);
   SDValue SoftenFloatRes_VAARG(SDNode *N);
   SDValue SoftenFloatRes_XINT_TO_FP(SDNode *N);
+  SDValue SoftenFloatRes_VECREDUCE(SDNode *N);
 
   // Convert Float Operand to Integer.
   bool SoftenFloatOperand(SDNode *N, unsigned OpNo);
   SDValue SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC);
   SDValue SoftenFloatOp_BITCAST(SDNode *N);
   SDValue SoftenFloatOp_BR_CC(SDNode *N);
   SDValue SoftenFloatOp_FP_ROUND(SDNode *N);
   SDValue SoftenFloatOp_FP_TO_XINT(SDNode *N);
private:
   SDValue PromoteFloatRes_FP_ROUND(SDNode *N);
   SDValue PromoteFloatRes_LOAD(SDNode *N);
   SDValue PromoteFloatRes_SELECT(SDNode *N);
   SDValue PromoteFloatRes_SELECT_CC(SDNode *N);
   SDValue PromoteFloatRes_UnaryOp(SDNode *N);
   SDValue PromoteFloatRes_UNDEF(SDNode *N);
   SDValue BitcastToInt_ATOMIC_SWAP(SDNode *N);
   SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N);
+  SDValue PromoteFloatRes_VECREDUCE(SDNode *N);
 
   bool PromoteFloatOperand(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo);

llvm/lib/Target/ARM/ARMTargetTransformInfo.h

public:
   bool preferPredicatedReductionSelect(unsigned Opcode, Type *Ty,
                                        TTI::ReductionFlags Flags) const;
 
   bool shouldExpandReduction(const IntrinsicInst *II) const {
     switch (II->getIntrinsicID()) {
     case Intrinsic::experimental_vector_reduce_v2_fadd:
     case Intrinsic::experimental_vector_reduce_v2_fmul:
       // We don't have legalization support for ordered FP reductions.
-      if (!II->getFastMathFlags().allowReassoc())
-        return true;
-      // Can't legalize reductions with soft floats.
-      return TLI->useSoftFloat() || !TLI->getSubtarget()->hasFPRegs();
-
-    case Intrinsic::experimental_vector_reduce_fmin:
-    case Intrinsic::experimental_vector_reduce_fmax:
-      // Can't legalize reductions with soft floats.
-      return TLI->useSoftFloat() || !TLI->getSubtarget()->hasFPRegs();
-
+      return !II->getFastMathFlags().allowReassoc();
     default:
       // Don't expand anything else, let legalization deal with it.
       return false;
     }
   }
 
   int getCFInstrCost(unsigned Opcode,
                      TTI::TargetCostKind CostKind);

llvm/test/CodeGen/ARM/vecreduce-fadd-legalization-soft-float.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm-none-eabi -mattr=-neon | FileCheck %s --check-prefix=CHECK
 
 declare half @llvm.experimental.vector.reduce.v2.fadd.f16.v4f16(half, <4 x half>)
 declare float @llvm.experimental.vector.reduce.v2.fadd.f32.v4f32(float, <4 x float>)
 declare double @llvm.experimental.vector.reduce.v2.fadd.f64.v2f64(double, <2 x double>)
 declare fp128 @llvm.experimental.vector.reduce.v2.fadd.f128.v2f128(fp128, <2 x fp128>)
 
 define half @test_v4f16(<4 x half> %a) nounwind {
 ; CHECK-LABEL: test_v4f16:
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    mov r7, #255
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    orr r7, r7, #65280
-; CHECK-NEXT:    mov r4, r3
-; CHECK-NEXT:    and r0, r1, r7
+; CHECK-NEXT:    mov r4, #255
+; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    orr r4, r4, #65280
 ; CHECK-NEXT:    mov r5, r2
+; CHECK-NEXT:    and r0, r3, r4
+; CHECK-NEXT:    mov r6, r1
 ; CHECK-NEXT:    bl __aeabi_h2f
 ; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    and r0, r4, r7
+; CHECK-NEXT:    and r0, r5, r4
 ; CHECK-NEXT:    bl __aeabi_h2f
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    and r0, r6, r7
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    and r0, r7, r4
 ; CHECK-NEXT:    bl __aeabi_h2f
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    and r0, r5, r7
+; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    and r0, r6, r4
 ; CHECK-NEXT:    bl __aeabi_h2f
 ; CHECK-NEXT:    mov r1, r0
-; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    bl __aeabi_fadd
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_fadd
-; CHECK-NEXT:    mov r1, r0
-; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r8
 ; CHECK-NEXT:    bl __aeabi_fadd
 ; CHECK-NEXT:    bl __aeabi_f2h
 ; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    mov pc, lr
   %b = call fast half @llvm.experimental.vector.reduce.v2.fadd.f16.v4f16(half 0.0, <4 x half> %a)
   ret half %b
 }
 
 define float @test_v4f32(<4 x float> %a) nounwind {
 ; CHECK-LABEL: test_v4f32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, lr}
-; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    mov r5, r1
-; CHECK-NEXT:    mov r1, r2
+; CHECK-NEXT:    .save {r4, r5, r11, lr}
+; CHECK-NEXT:    push {r4, r5, r11, lr}
 ; CHECK-NEXT:    mov r4, r3
+; CHECK-NEXT:    mov r5, r2
 ; CHECK-NEXT:    bl __aeabi_fadd
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r1, r5
 ; CHECK-NEXT:    bl __aeabi_fadd
-; CHECK-NEXT:    mov r1, r0
-; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    bl __aeabi_fadd
-; CHECK-NEXT:    pop {r4, r5, r6, lr}
+; CHECK-NEXT:    pop {r4, r5, r11, lr}
 ; CHECK-NEXT:    mov pc, lr
   %b = call fast float @llvm.experimental.vector.reduce.v2.fadd.f32.v4f32(float 0.0, <4 x float> %a)
   ret float %b
 }
 
 define double @test_v2f64(<2 x double> %a) nounwind {
 ; CHECK-LABEL: test_v2f64:
 ; CHECK:       @ %bb.0: