[AArch64] Generate FADDP from shuffled fadd

[dmgreen]

Summary

As a follow up to D126686, this does the same fold for floating point add and shuffle. In this case it is limited to reassoc either x[0]+x[1] or x[1]+x[0] for both result[0] and results[1].

[samtebbs]

lgtm

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

if (Subtarget->hasNEON()) {
     setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Custom);
     setLoadExtAction(ISD::EXTLOAD,  MVT::v4i32, MVT::v4i8, Custom);
     setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Custom);
     setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Custom);
 
     // ADDP custom lowering
     for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
       setOperationAction(ISD::ADD, VT, Custom);
+    // FADDP custom lowering
+    for (MVT VT : { MVT::v16f16, MVT::v8f32, MVT::v4f64 })
+      setOperationAction(ISD::FADD, VT, Custom);
   }
 
   if (Subtarget->hasSVE()) {
     for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) {
       setOperationAction(ISD::BITREVERSE, VT, Custom);
       setOperationAction(ISD::BSWAP, VT, Custom);
       setOperationAction(ISD::CTLZ, VT, Custom);
       setOperationAction(ISD::CTPOP, VT, Custom);
Op = SDValue(
                          DAG.getUNDEF(MVT::i32), Op,
                          DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
       0);
   Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op);
   Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op));
 }
 
 static void ReplaceAddWithADDP(SDNode *N, SmallVectorImpl<SDValue> &Results,
-                               SelectionDAG &DAG) {
+                               SelectionDAG &DAG,
+                               const AArch64Subtarget *Subtarget) {
   EVT VT = N->getValueType(0);
-  if (!VT.is256BitVector())
+  if (!VT.is256BitVector() ||
+      (VT.getScalarType().isFloatingPoint() &&
+       !N->getFlags().hasAllowReassociation()) ||
+      (VT.getScalarType() == MVT::f16 && !Subtarget->hasFullFP16()))
     return;
 
   SDValue X = N->getOperand(0);
   auto *Shuf = dyn_cast<ShuffleVectorSDNode>(N->getOperand(1));
   if (!Shuf) {
     Shuf = dyn_cast<ShuffleVectorSDNode>(N->getOperand(0));
     X = N->getOperand(1);
     if (!Shuf)
void AArch64TargetLowering::ReplaceNodeResults(
   case ISD::VECREDUCE_ADD:
   case ISD::VECREDUCE_SMAX:
   case ISD::VECREDUCE_SMIN:
   case ISD::VECREDUCE_UMAX:
   case ISD::VECREDUCE_UMIN:
     Results.push_back(LowerVECREDUCE(SDValue(N, 0), DAG));
     return;
   case ISD::ADD:
-    ReplaceAddWithADDP(N, Results, DAG);
+  case ISD::FADD:
+    ReplaceAddWithADDP(N, Results, DAG, Subtarget);
     return;
 
   case ISD::CTPOP:
     if (SDValue Result = LowerCTPOP(SDValue(N, 0), DAG))
       Results.push_back(Result);
     return;
   case AArch64ISD::SADDV:
     ReplaceReductionResults(N, Results, DAG, ISD::ADD, AArch64ISD::SADDV);

llvm/lib/Target/AArch64/AArch64InstrInfo.td

def AArch64addp     : PatFrags<(ops node:$Rn, node:$Rm),
                                [(AArch64addp_n node:$Rn, node:$Rm),
                                 (int_aarch64_neon_addp node:$Rn, node:$Rm)]>;
 def AArch64uaddlp   : PatFrags<(ops node:$src),
                                [(AArch64uaddlp_n node:$src),
                                 (int_aarch64_neon_uaddlp node:$src)]>;
 def AArch64saddlp   : PatFrags<(ops node:$src),
                                [(AArch64saddlp_n node:$src),
                                 (int_aarch64_neon_saddlp node:$src)]>;
+def AArch64faddp     : PatFrags<(ops node:$Rn, node:$Rm),
+                                [(AArch64addp_n node:$Rn, node:$Rm),
+                                 (int_aarch64_neon_faddp node:$Rn, node:$Rm)]>;
 
 def SDT_AArch64SETTAG : SDTypeProfile<0, 2, [SDTCisPtrTy<0>, SDTCisPtrTy<1>]>;
 def AArch64stg : SDNode<"AArch64ISD::STG", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def AArch64stzg : SDNode<"AArch64ISD::STZG", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def AArch64st2g : SDNode<"AArch64ISD::ST2G", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def AArch64stz2g : SDNode<"AArch64ISD::STZ2G", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 
 def SDT_AArch64unpk : SDTypeProfile<1, 1, [

 def : Pat<(fabs (fsub VT:$Rn, VT:$Rm)), (!cast<Instruction>("FABD"#VT) VT:$Rn, VT:$Rm)>;
 }
 let Predicates = [HasNEON, HasFullFP16] in {
 foreach VT = [ v4f16, v8f16 ] in
 def : Pat<(fabs (fsub VT:$Rn, VT:$Rm)), (!cast<Instruction>("FABD"#VT) VT:$Rn, VT:$Rm)>;
 }
 defm FACGE   : SIMDThreeSameVectorFPCmp<1,0,0b101,"facge",int_aarch64_neon_facge>;
 defm FACGT   : SIMDThreeSameVectorFPCmp<1,1,0b101,"facgt",int_aarch64_neon_facgt>;
-defm FADDP   : SIMDThreeSameVectorFP<1,0,0b010,"faddp",int_aarch64_neon_faddp>;
+defm FADDP   : SIMDThreeSameVectorFP<1,0,0b010,"faddp", AArch64faddp>;
 defm FADD    : SIMDThreeSameVectorFP<0,0,0b010,"fadd", any_fadd>;
 defm FCMEQ   : SIMDThreeSameVectorFPCmp<0, 0, 0b100, "fcmeq", AArch64fcmeq>;
 defm FCMGE   : SIMDThreeSameVectorFPCmp<1, 0, 0b100, "fcmge", AArch64fcmge>;
 defm FCMGT   : SIMDThreeSameVectorFPCmp<1, 1, 0b100, "fcmgt", AArch64fcmgt>;
 defm FDIV    : SIMDThreeSameVectorFP<1,0,0b111,"fdiv", any_fdiv>;
 defm FMAXNMP : SIMDThreeSameVectorFP<1,0,0b000,"fmaxnmp", int_aarch64_neon_fmaxnmp>;
 defm FMAXNM  : SIMDThreeSameVectorFP<0,0,0b000,"fmaxnm", any_fmaxnum>;
 defm FMAXP   : SIMDThreeSameVectorFP<1,0,0b110,"fmaxp", int_aarch64_neon_fmaxp>;

llvm/test/CodeGen/AArch64/faddp-half.ll

entry:
   %s = shufflevector <8 x half> %a, <8 x half> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   %b = fadd reassoc <8 x half> %s, %a
   ret <8 x half> %b
 }
 
 define <16 x half> @addp_v16f16(<16 x half> %a) {
 ; CHECK-LABEL: addp_v16f16:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev32 v2.8h, v0.8h
-; CHECK-NEXT:    rev32 v3.8h, v1.8h
-; CHECK-NEXT:    fadd v0.8h, v2.8h, v0.8h
-; CHECK-NEXT:    fadd v1.8h, v3.8h, v1.8h
+; CHECK-NEXT:    faddp v1.8h, v0.8h, v1.8h
+; CHECK-NEXT:    zip1 v0.8h, v1.8h, v1.8h
+; CHECK-NEXT:    zip2 v1.8h, v1.8h, v1.8h
 ; CHECK-NEXT:    ret
 ;
 ; CHECKNOFP16-LABEL: addp_v16f16:
 ; CHECKNOFP16:       // %bb.0: // %entry
 ; CHECKNOFP16-NEXT:    rev32 v5.8h, v0.8h
 ; CHECKNOFP16-NEXT:    mov h6, v0.h[1]
 ; CHECKNOFP16-NEXT:    rev32 v4.8h, v1.8h
 ; CHECKNOFP16-NEXT:    fcvt s2, h0

llvm/test/CodeGen/AArch64/faddp.ll

entry:
   %s = shufflevector <2 x double> %a, <2 x double> poison, <2 x i32> <i32 1, i32 0>
   %b = fadd reassoc <2 x double> %s, %a
   ret <2 x double> %b
 }
 
 define <4 x double> @addp_v4f64(<4 x double> %a) {
 ; CHECK-LABEL: addp_v4f64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    fadd v0.2d, v2.2d, v0.2d
-; CHECK-NEXT:    fadd v1.2d, v3.2d, v1.2d
+; CHECK-NEXT:    faddp v1.2d, v0.2d, v1.2d
+; CHECK-NEXT:    dup v0.2d, v1.d[0]
+; CHECK-NEXT:    dup v1.2d, v1.d[1]
 ; CHECK-NEXT:    ret
 entry:
   %s = shufflevector <4 x double> %a, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   %b = fadd reassoc <4 x double> %s, %a
   ret <4 x double> %b
 }
 
 define <4 x float> @addp_v4f32(<4 x float> %a) {
entry:
   %s = shufflevector <8 x float> %a, <8 x float> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   %b = fadd <8 x float> %s, %a
   ret <8 x float> %b
 }
 
 define <8 x float> @addp_v8f32_slow(<8 x float> %a) {
 ; CHECK-LABEL: addp_v8f32_slow:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev64 v2.4s, v0.4s
-; CHECK-NEXT:    rev64 v3.4s, v1.4s
-; CHECK-NEXT:    fadd v0.4s, v2.4s, v0.4s
-; CHECK-NEXT:    fadd v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    faddp v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    zip1 v0.4s, v1.4s, v1.4s
+; CHECK-NEXT:    zip2 v1.4s, v1.4s, v1.4s
 ; CHECK-NEXT:    ret
 entry:
   %s = shufflevector <8 x float> %a, <8 x float> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   %b = fadd reassoc <8 x float> %s, %a
   ret <8 x float> %b
 }
 
 define <16 x float> @addp_v16f32(<16 x float> %a) {
 ; CHECK-LABEL: addp_v16f32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev64 v4.4s, v0.4s
-; CHECK-NEXT:    rev64 v5.4s, v1.4s
-; CHECK-NEXT:    rev64 v6.4s, v2.4s
-; CHECK-NEXT:    rev64 v7.4s, v3.4s
-; CHECK-NEXT:    fadd v0.4s, v4.4s, v0.4s
-; CHECK-NEXT:    fadd v1.4s, v5.4s, v1.4s
-; CHECK-NEXT:    fadd v2.4s, v6.4s, v2.4s
-; CHECK-NEXT:    fadd v3.4s, v7.4s, v3.4s
+; CHECK-NEXT:    faddp v3.4s, v2.4s, v3.4s
+; CHECK-NEXT:    faddp v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    zip1 v2.4s, v3.4s, v3.4s
+; CHECK-NEXT:    zip1 v0.4s, v1.4s, v1.4s
+; CHECK-NEXT:    zip2 v1.4s, v1.4s, v1.4s
+; CHECK-NEXT:    zip2 v3.4s, v3.4s, v3.4s
 ; CHECK-NEXT:    ret
 entry:
   %s = shufflevector <16 x float> %a, <16 x float> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
   %b = fadd reassoc <16 x float> %s, %a
   ret <16 x float> %b
 }
 
 
 attributes #0 = { strictfp }
 
 declare <2 x float> @llvm.experimental.constrained.fadd.v2f32(<2 x float>, <2 x float>, metadata, metadata)
 declare <4 x float> @llvm.experimental.constrained.fadd.v4f32(<4 x float>, <4 x float>, metadata, metadata)
 declare <2 x double> @llvm.experimental.constrained.fadd.v2f64(<2 x double>, <2 x double>, metadata, metadata)