X86-specific path: Implemented the fusing of MUL+SUBADD to FMSUBADD

[Quolyk]

Summary

This is my first attempt to contribute to llvm. I'm trying to implement this https://bugs.llvm.org/show_bug.cgi?id=30633. MUL+ADDSUB to FMADDSUB was implemented here https://reviews.llvm.org/D28087. I'm looking for help now.

[craig.topper]

lib/Target/X86/X86ISelLowering.cpp

7523

Rename to isFMAddSubOrFMSubAdd?

7531

Should just return SDValue() here no matter the type. We can't create a X86ISD::SUBADD, there's no such instruction.

29607

Maybe merge this with isAddSub by passing a bool value to indicate which one you're looking for. Then you just need to factor that bool into the code that detects the FSUB/FADD

lib/Target/X86/X86ISelLowering.h

204 ↗

(On Diff #123884)

There's no such instruction as SUBADD that doesn't include the multiply. So we shouldn't be adding this opcode.

[Quolyk]

Updated diff according to code review. I need some help with documenting updated functions

[craig.topper]

Is this really testing both ways of doing this? There's one from shuffles and one from build_vector.

According to the coverage build bot, the build_vector path of fmaddsub isn't tested today.

http://lab.llvm.org:8080/coverage/coverage-reports/llvm/coverage/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/X86/X86ISelLowering.cpp.html#L7505

lib/Target/X86/X86ISelLowering.cpp

7325–7326

Rename to isAddSubOrSubAdd?

7327

Call this matchSubAdd instead of isSubAdd.

29730

This should probably be "ADDSUB, FMADDSUB, or FMSUBADD"

[Quolyk]

lib/Target/X86/X86ISelLowering.cpp

7862

I added that if statement, but I have problems testing this, specifically I don't know how to test this

[Quolyk]

In D40335#935081, @craig.topper wrote:

Is this really testing both ways of doing this? There's one from shuffles and one from build_vector.

According to the coverage build bot, the build_vector path of fmaddsub isn't tested today.

http://lab.llvm.org:8080/coverage/coverage-reports/llvm/coverage/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/X86/X86ISelLowering.cpp.html#L7505

Lol, I added inline comment to that code, but didn't submit comment. Indeed, this isn't tested and I don't know how to do it.

[craig.topper]

Can you just remove the untested part? I'll add a TODO to the existing untested code indicating that it isn't tested.

[craig.topper]

LGTM

[Quolyk]

I don't have commit access, please commit.

[Quolyk]

@craig.topper please commit

[craig.topper]

Sorry missed your previous comment. Will do.

lib/Target/X86/X86ISelLowering.cpp

if (Mode) {
 
     if (!isUndefHI && (!V0_HI->isUndef() || !V1_HI->isUndef()))
       HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI);
   }
 
   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI);
 }
 
 /// Returns true iff \p BV builds a vector with the result equivalent to
-/// the result of ADDSUB operation.
-/// If true is returned then the operands of ADDSUB = Opnd0 +- Opnd1 operation
-/// are written to the parameters \p Opnd0 and \p Opnd1.
-static bool isAddSub(const BuildVectorSDNode *BV,
+/// the result of ADDSUB(SUBADD) operation.
+/// If true is returned then the operands of ADDSUB = Opnd0 +- Opnd1
+/// (SUBADD = Opnd0 -+ Opnd1) operation are written to the 
+/// parameters \p Opnd0 and \p Opnd1.
+static bool isAddSubOrSubAdd(const BuildVectorSDNode *BV,
                              const X86Subtarget &Subtarget, SelectionDAG &DAG,
-                     SDValue &Opnd0, SDValue &Opnd1) {
+                             SDValue &Opnd0, SDValue &Opnd1,
+                             bool matchSubAdd = false) {
 
   MVT VT = BV->getSimpleValueType(0);
   if ((!Subtarget.hasSSE3() || (VT != MVT::v4f32 && VT != MVT::v2f64)) &&
       (!Subtarget.hasAVX() || (VT != MVT::v8f32 && VT != MVT::v4f64)) &&
       (!Subtarget.hasAVX512() || (VT != MVT::v16f32 && VT != MVT::v8f64)))
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
   SDValue InVec0 = DAG.getUNDEF(VT);
   SDValue InVec1 = DAG.getUNDEF(VT);
 
   // Odd-numbered elements in the input build vector are obtained from
   // adding two integer/float elements.
   // Even-numbered elements in the input build vector are obtained from
   // subtracting two integer/float elements.
-  unsigned ExpectedOpcode = ISD::FSUB;
-  unsigned NextExpectedOpcode = ISD::FADD;
+  unsigned ExpectedOpcode = matchSubAdd ? ISD::FADD : ISD::FSUB;
+  unsigned NextExpectedOpcode = matchSubAdd ? ISD::FSUB : ISD::FADD;
   bool AddFound = false;
   bool SubFound = false;
 
   for (unsigned i = 0, e = NumElts; i != e; ++i) {
     SDValue Op = BV->getOperand(i);
 
     // Skip 'undef' values.
     unsigned Opcode = Op.getOpcode();
if (!AddFound || !SubFound || InVec0.isUndef() || InVec1.isUndef())
     return false;
 
   Opnd0 = InVec0;
   Opnd1 = InVec1;
   return true;
 }
 
 /// Returns true if is possible to fold MUL and an idiom that has already been
-/// recognized as ADDSUB(\p Opnd0, \p Opnd1) into FMADDSUB(x, y, \p Opnd1).
-/// If (and only if) true is returned, the operands of FMADDSUB are written to
-/// parameters \p Opnd0, \p Opnd1, \p Opnd2.
+/// recognized as ADDSUB/SUBADD(\p Opnd0, \p Opnd1) into 
+/// FMADDSUB/FMSUBADD(x, y, \p Opnd1). If (and only if) true is returned, the
+/// operands of FMADDSUB/FMSUBADD are written to parameters \p Opnd0, \p Opnd1, \p Opnd2.
 ///
 /// Prior to calling this function it should be known that there is some
 /// SDNode that potentially can be replaced with an X86ISD::ADDSUB operation
 /// using \p Opnd0 and \p Opnd1 as operands. Also, this method is called
 /// before replacement of such SDNode with ADDSUB operation. Thus the number
 /// of \p Opnd0 uses is expected to be equal to 2.
 /// For example, this function may be called for the following IR:
 ///    %AB = fmul fast <2 x double> %A, %B
 ///    %Sub = fsub fast <2 x double> %AB, %C
 ///    %Add = fadd fast <2 x double> %AB, %C
 ///    %Addsub = shufflevector <2 x double> %Sub, <2 x double> %Add,
 ///                            <2 x i32> <i32 0, i32 3>
 /// There is a def for %Addsub here, which potentially can be replaced by
 /// X86ISD::ADDSUB operation:
 ///    %Addsub = X86ISD::ADDSUB %AB, %C
 /// and such ADDSUB can further be replaced with FMADDSUB:
 ///    %Addsub = FMADDSUB %A, %B, %C.
 ///
 /// The main reason why this method is called before the replacement of the
 /// recognized ADDSUB idiom with ADDSUB operation is that such replacement
 /// is illegal sometimes. E.g. 512-bit ADDSUB is not available, while 512-bit
 /// FMADDSUB is.
-static bool isFMAddSub(const X86Subtarget &Subtarget, SelectionDAG &DAG,
+static bool isFMAddSubOrFMSubAdd(const X86Subtarget &Subtarget, SelectionDAG &DAG,
                                  SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2) {
   if (Opnd0.getOpcode() != ISD::FMUL || Opnd0->use_size() != 2 ||
       !Subtarget.hasAnyFMA())
     return false;
 
   // FIXME: These checks must match the similar ones in
   // DAGCombiner::visitFADDForFMACombine. It would be good to have one
   // function that would answer if it is Ok to fuse MUL + ADD to FMADD
   // or MUL + ADDSUB to FMADDSUB.

 }
 
 /// Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' operation
 /// accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB node.
 static SDValue lowerToAddSubOrFMAddSub(const BuildVectorSDNode *BV,
                                        const X86Subtarget &Subtarget,
                                        SelectionDAG &DAG) {
   SDValue Opnd0, Opnd1;
-  if (!isAddSub(BV, Subtarget, DAG, Opnd0, Opnd1))
+  if (!isAddSubOrSubAdd(BV, Subtarget, DAG, Opnd0, Opnd1))
     return SDValue();
 
   MVT VT = BV->getSimpleValueType(0);
   SDLoc DL(BV);
 
   // Try to generate X86ISD::FMADDSUB node here.
   SDValue Opnd2;
-  if (isFMAddSub(Subtarget, DAG, Opnd0, Opnd1, Opnd2))
+  if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2))
     return DAG.getNode(X86ISD::FMADDSUB, DL, VT, Opnd0, Opnd1, Opnd2);
 
   // Do not generate X86ISD::ADDSUB node for 512-bit types even though
   // the ADDSUB idiom has been successfully recognized. There are no known
   // X86 targets with 512-bit ADDSUB instructions!
   // 512-bit ADDSUB idiom recognition was needed only as part of FMADDSUB idiom
   // recognition.
   if (VT.is512BitVector())
     return SDValue();
 
   return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
 }
 
+/// Try to fold a build_vector that performs an 'fmsubadd' operation
+/// to X86ISD::FMSUBADD node.
+static SDValue lowerToFMSubAdd(const BuildVectorSDNode *BV,
+                               const X86Subtarget &Subtarget,
+                               SelectionDAG &DAG) {
+  SDValue Opnd0, Opnd1;
+  if (!isAddSubOrSubAdd(BV, Subtarget, DAG, Opnd0, Opnd1, true))
+    return SDValue();
+  
+  MVT VT = BV->getSimpleValueType(0);
+  SDLoc DL(BV);
+  
+  // Try to generate X86ISD::FMSUBADD node here.
+  SDValue Opnd2;
+  if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2))
+    return DAG.getNode(X86ISD::FMSUBADD, DL, VT, Opnd0, Opnd1, Opnd2);
+  
+  return SDValue();
+}
+
 /// Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
 static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV,
                                    const X86Subtarget &Subtarget,
                                    SelectionDAG &DAG) {
   MVT VT = BV->getSimpleValueType(0);
   unsigned NumElts = VT.getVectorNumElements();
   unsigned NumUndefsLO = 0;
   unsigned NumUndefsHI = 0;
   unsigned Half = NumElts/2;
 
   // Count the number of UNDEF operands in the build_vector in input.
if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512())
     return LowerBUILD_VECTORvXi1(Op, DAG);
 
   if (SDValue VectorConstant = materializeVectorConstant(Op, DAG, Subtarget))
     return VectorConstant;
 
   BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Op.getNode());
   if (SDValue AddSub = lowerToAddSubOrFMAddSub(BV, Subtarget, DAG))
     return AddSub;
+  if (SDValue FMSubAdd = lowerToFMSubAdd(BV, Subtarget, DAG))
+    return FMSubAdd;
   if (SDValue HorizontalOp = LowerToHorizontalOp(BV, Subtarget, DAG))
     return HorizontalOp;
   if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, Subtarget, DAG))
     return Broadcast;
   if (SDValue BitOp = lowerBuildVectorToBitOp(BV, DAG))
     return BitOp;
 
   unsigned EVTBits = ExtVT.getSizeInBits();
if (SDValue NewN = combineRedundantDWordShuffle(N, Mask, DAG))
       return NewN;
 
     break;
   }
 
   return SDValue();
 }
 
-/// Returns true iff the shuffle node \p N can be replaced with ADDSUB
-/// operation. If true is returned then the operands of ADDSUB operation
+/// Returns true iff the shuffle node \p N can be replaced with ADDSUB(SUBADD)
+/// operation. If true is returned then the operands of ADDSUB(SUBADD) operation
 /// are written to the parameters \p Opnd0 and \p Opnd1.
 ///
-/// We combine shuffle to ADDSUB directly on the abstract vector shuffle nodes
+/// We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle nodes
 /// so it is easier to generically match. We also insert dummy vector shuffle
 /// nodes for the operands which explicitly discard the lanes which are unused
 /// by this operation to try to flow through the rest of the combiner
 /// the fact that they're unused.
-static bool isAddSub(SDNode *N, const X86Subtarget &Subtarget,
-                     SDValue &Opnd0, SDValue &Opnd1) {
+static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
+                             SDValue &Opnd0, SDValue &Opnd1, 
+                             bool matchSubAdd = false) {
 
   EVT VT = N->getValueType(0);
   if ((!Subtarget.hasSSE3() || (VT != MVT::v4f32 && VT != MVT::v2f64)) &&
       (!Subtarget.hasAVX() || (VT != MVT::v8f32 && VT != MVT::v4f64)) &&
       (!Subtarget.hasAVX512() || (VT != MVT::v16f32 && VT != MVT::v8f64)))
     return false;
 
   // We only handle target-independent shuffles.
   // FIXME: It would be easy and harmless to use the target shuffle mask
   // extraction tool to support more.
   if (N->getOpcode() != ISD::VECTOR_SHUFFLE)
     return false;
 
   ArrayRef<int> OrigMask = cast<ShuffleVectorSDNode>(N)->getMask();
   SmallVector<int, 16> Mask(OrigMask.begin(), OrigMask.end());
 
   SDValue V1 = N->getOperand(0);
   SDValue V2 = N->getOperand(1);
   
-  // We require the first shuffle operand to be the FSUB node, and the second to
-  // be the FADD node.
-  if (V1.getOpcode() == ISD::FADD && V2.getOpcode() == ISD::FSUB) {
+  unsigned ExpectedOpcode = matchSubAdd ? ISD::FADD : ISD::FSUB;
+  unsigned NextExpectedOpcode = matchSubAdd ? ISD::FSUB : ISD::FADD;
+
+  // We require the first shuffle operand to be the ExpectedOpcode node,
+  // and the second to be the NextExpectedOpcode node.
+  if (V1.getOpcode() == NextExpectedOpcode && V2.getOpcode() == ExpectedOpcode) {
     ShuffleVectorSDNode::commuteMask(Mask);
     std::swap(V1, V2);
-  } else if (V1.getOpcode() != ISD::FSUB || V2.getOpcode() != ISD::FADD)
+  } else if (V1.getOpcode() != ExpectedOpcode || V2.getOpcode() != NextExpectedOpcode)
     return false;
 
   // If there are other uses of these operations we can't fold them.
   if (!V1->hasOneUse() || !V2->hasOneUse())
     return false;
 
   // Ensure that both operations have the same operands. Note that we can
   // commute the FADD operands.

 }
 
 /// \brief Try to combine a shuffle into a target-specific add-sub or
 /// mul-add-sub node.
 static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N,
                                                 const X86Subtarget &Subtarget,
                                                 SelectionDAG &DAG) {
   SDValue Opnd0, Opnd1;
-  if (!isAddSub(N, Subtarget, Opnd0, Opnd1))
+  if (!isAddSubOrSubAdd(N, Subtarget, Opnd0, Opnd1))
     return SDValue();
 
   EVT VT = N->getValueType(0);
   SDLoc DL(N);
 
   // Try to generate X86ISD::FMADDSUB node here.
   SDValue Opnd2;
-  if (isFMAddSub(Subtarget, DAG, Opnd0, Opnd1, Opnd2))
+  if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2))
     return DAG.getNode(X86ISD::FMADDSUB, DL, VT, Opnd0, Opnd1, Opnd2);
 
   // Do not generate X86ISD::ADDSUB node for 512-bit types even though
   // the ADDSUB idiom has been successfully recognized. There are no known
   // X86 targets with 512-bit ADDSUB instructions!
   if (VT.is512BitVector())
     return SDValue();
 
   return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
 }
 
+/// \brief Try to combine a shuffle into a target-specific
+/// mul-sub-add node.
+static SDValue combineShuffleToFMSubAdd(SDNode *N,
+                                        const X86Subtarget &Subtarget,
+                                        SelectionDAG &DAG) {
+  SDValue Opnd0, Opnd1;
+  if (!isAddSubOrSubAdd(N, Subtarget, Opnd0, Opnd1, true))
+    return SDValue();
+  
+  EVT VT = N->getValueType(0);
+  SDLoc DL(N);
+  
+  // Try to generate X86ISD::FMSUBADD node here.
+  SDValue Opnd2;
+  if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2))
+    return DAG.getNode(X86ISD::FMSUBADD, DL, VT, Opnd0, Opnd1, Opnd2);
+  
+  return SDValue();
+}
+
 // We are looking for a shuffle where both sources are concatenated with undef
 // and have a width that is half of the output's width. AVX2 has VPERMD/Q, so
 // if we can express this as a single-source shuffle, that's preferable.
 static SDValue combineShuffleOfConcatUndef(SDNode *N, SelectionDAG &DAG,
                                            const X86Subtarget &Subtarget) {
   if (!Subtarget.hasAVX2() || !isa<ShuffleVectorSDNode>(N))
     return SDValue();
 

 
 static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
                               TargetLowering::DAGCombinerInfo &DCI,
                               const X86Subtarget &Subtarget) {
   SDLoc dl(N);
   EVT VT = N->getValueType(0);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   // If we have legalized the vector types, look for blends of FADD and FSUB
-  // nodes that we can fuse into an ADDSUB node.
+  // nodes that we can fuse into an ADDSUB, FMADDSUB, or FMSUBADD node.
   if (TLI.isTypeLegal(VT)) {
     if (SDValue AddSub = combineShuffleToAddSubOrFMAddSub(N, Subtarget, DAG))
       return AddSub;
     
+    if (SDValue FMSubAdd = combineShuffleToFMSubAdd(N, Subtarget, DAG))
+      return FMSubAdd;
+
     if (SDValue HAddSub = foldShuffleOfHorizOp(N))
       return HAddSub;
   }
 
   // During Type Legalization, when promoting illegal vector types,
   // the backend might introduce new shuffle dag nodes and bitcasts.
   //
   // This code performs the following transformation:

test/CodeGen/X86/fmsubadd-combine.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+fma | FileCheck -check-prefix=FMA3 -check-prefix=FMA3_256 %s
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+fma,+avx512f | FileCheck -check-prefix=FMA3 -check-prefix=FMA3_512 %s
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+fma4 | FileCheck -check-prefix=FMA4 %s
 
 ; This test checks the fusing of MUL + SUB/ADD to FMSUBADD.
 
 define <2 x double> @mul_subadd_pd128(<2 x double> %A, <2 x double> %B, <2 x double> %C) #0 {
 ; FMA3_256-LABEL: mul_subadd_pd128:
 ; FMA3_256:       # BB#0: # %entry
-; FMA3_256-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
-; FMA3_256-NEXT:    vsubpd %xmm2, %xmm0, %xmm1
-; FMA3_256-NEXT:    vaddpd %xmm2, %xmm0, %xmm0
-; FMA3_256-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; FMA3_256-NEXT:    vfmsubadd213pd %xmm2, %xmm1, %xmm0
 ; FMA3_256-NEXT:    retq
 ;
 ; FMA3_512-LABEL: mul_subadd_pd128:
 ; FMA3_512:       # BB#0: # %entry
-; FMA3_512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
-; FMA3_512-NEXT:    vsubpd %xmm2, %xmm0, %xmm1
-; FMA3_512-NEXT:    vaddpd %xmm2, %xmm0, %xmm0
-; FMA3_512-NEXT:    vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; FMA3_512-NEXT:    vfmsubadd213pd %xmm2, %xmm1, %xmm0
 ; FMA3_512-NEXT:    retq
 ;
 ; FMA4-LABEL: mul_subadd_pd128:
 ; FMA4:       # BB#0: # %entry
-; FMA4-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
-; FMA4-NEXT:    vsubpd %xmm2, %xmm0, %xmm1
-; FMA4-NEXT:    vaddpd %xmm2, %xmm0, %xmm0
-; FMA4-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],xmm1[1]
+; FMA4-NEXT:    vfmsubaddpd %xmm2, %xmm1, %xmm0, %xmm0
 ; FMA4-NEXT:    retq
 entry:
   %AB = fmul <2 x double> %A, %B
   %Sub = fsub <2 x double> %AB, %C
   %Add = fadd <2 x double> %AB, %C
   %subadd = shufflevector <2 x double> %Add, <2 x double> %Sub, <2 x i32> <i32 0, i32 3>
   ret <2 x double> %subadd
 }
 
 define <4 x float> @mul_subadd_ps128(<4 x float> %A, <4 x float> %B, <4 x float> %C) #0 {
 ; FMA3-LABEL: mul_subadd_ps128:
 ; FMA3:       # BB#0: # %entry
-; FMA3-NEXT:    vmulps %xmm1, %xmm0, %xmm0
-; FMA3-NEXT:    vsubps %xmm2, %xmm0, %xmm1
-; FMA3-NEXT:    vaddps %xmm2, %xmm0, %xmm0
-; FMA3-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; FMA3-NEXT:    vfmsubadd213ps  %xmm2, %xmm1, %xmm0
 ; FMA3-NEXT:    retq
 ;
 ; FMA4-LABEL: mul_subadd_ps128:
 ; FMA4:       # BB#0: # %entry
-; FMA4-NEXT:    vmulps %xmm1, %xmm0, %xmm0
-; FMA4-NEXT:    vsubps %xmm2, %xmm0, %xmm1
-; FMA4-NEXT:    vaddps %xmm2, %xmm0, %xmm0
-; FMA4-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; FMA4-NEXT:    vfmsubaddps %xmm2, %xmm1, %xmm0, %xmm0
 ; FMA4-NEXT:    retq
 entry:
   %AB = fmul <4 x float> %A, %B
   %Sub = fsub <4 x float> %AB, %C
   %Add = fadd <4 x float> %AB, %C
   %subadd = shufflevector <4 x float> %Add, <4 x float> %Sub, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
   ret <4 x float> %subadd
 }
 
 define <4 x double> @mul_subadd_pd256(<4 x double> %A, <4 x double> %B, <4 x double> %C) #0 {
 ; FMA3-LABEL: mul_subadd_pd256:
 ; FMA3:       # BB#0: # %entry
-; FMA3-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
-; FMA3-NEXT:    vsubpd %ymm2, %ymm0, %ymm1
-; FMA3-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
-; FMA3-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
+; FMA3-NEXT:    vfmsubadd213pd  %ymm2, %ymm1, %ymm0
 ; FMA3-NEXT:    retq
 ;
 ; FMA4-LABEL: mul_subadd_pd256:
 ; FMA4:       # BB#0: # %entry
-; FMA4-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
-; FMA4-NEXT:    vsubpd %ymm2, %ymm0, %ymm1
-; FMA4-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
-; FMA4-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
+; FMA4-NEXT:    vfmsubaddpd %ymm2, %ymm1, %ymm0, %ymm0
 ; FMA4-NEXT:    retq
 entry:
   %AB = fmul <4 x double> %A, %B
   %Sub = fsub <4 x double> %AB, %C
   %Add = fadd <4 x double> %AB, %C
   %subadd = shufflevector <4 x double> %Add, <4 x double> %Sub, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
   ret <4 x double> %subadd
 }
 
 define <8 x float> @mul_subadd_ps256(<8 x float> %A, <8 x float> %B, <8 x float> %C) #0 {
 ; FMA3-LABEL: mul_subadd_ps256:
 ; FMA3:       # BB#0: # %entry
-; FMA3-NEXT:    vmulps %ymm1, %ymm0, %ymm0
-; FMA3-NEXT:    vsubps %ymm2, %ymm0, %ymm1
-; FMA3-NEXT:    vaddps %ymm2, %ymm0, %ymm0
-; FMA3-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
+; FMA3-NEXT:    vfmsubadd213ps  %ymm2, %ymm1, %ymm0
 ; FMA3-NEXT:    retq
 ;
 ; FMA4-LABEL: mul_subadd_ps256:
 ; FMA4:       # BB#0: # %entry
-; FMA4-NEXT:    vmulps %ymm1, %ymm0, %ymm0
-; FMA4-NEXT:    vsubps %ymm2, %ymm0, %ymm1
-; FMA4-NEXT:    vaddps %ymm2, %ymm0, %ymm0
-; FMA4-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
+; FMA4-NEXT:    vfmsubaddps %ymm2, %ymm1, %ymm0, %ymm0
 ; FMA4-NEXT:    retq
 entry:
   %AB = fmul <8 x float> %A, %B
   %Sub = fsub <8 x float> %AB, %C
   %Add = fadd <8 x float> %AB, %C
   %subadd = shufflevector <8 x float> %Add, <8 x float> %Sub, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
   ret <8 x float> %subadd
 }
 
 define <8 x double> @mul_subadd_pd512(<8 x double> %A, <8 x double> %B, <8 x double> %C) #0 {
 ; FMA3_256-LABEL: mul_subadd_pd512:
 ; FMA3_256:       # BB#0: # %entry
-; FMA3_256-NEXT:    vmulpd %ymm2, %ymm0, %ymm0
-; FMA3_256-NEXT:    vmulpd %ymm3, %ymm1, %ymm1
-; FMA3_256-NEXT:    vsubpd %ymm5, %ymm1, %ymm2
-; FMA3_256-NEXT:    vsubpd %ymm4, %ymm0, %ymm3
-; FMA3_256-NEXT:    vaddpd %ymm5, %ymm1, %ymm1
-; FMA3_256-NEXT:    vblendpd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3]
-; FMA3_256-NEXT:    vaddpd %ymm4, %ymm0, %ymm0
-; FMA3_256-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2],ymm3[3]
+; FMA3_256-NEXT:    vfmsubadd213pd  %ymm4, %ymm2, %ymm0
+; FMA3_256-NEXT:    vfmsubadd213pd  %ymm5, %ymm3, %ymm1
 ; FMA3_256-NEXT:    retq
 ;
 ; FMA3_512-LABEL: mul_subadd_pd512:
 ; FMA3_512:       # BB#0: # %entry
-; FMA3_512-NEXT:    vmulpd %zmm1, %zmm0, %zmm0
-; FMA3_512-NEXT:    vsubpd %zmm2, %zmm0, %zmm1
-; FMA3_512-NEXT:    vaddpd %zmm2, %zmm0, %zmm0
-; FMA3_512-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[0],zmm1[1],zmm0[2],zmm1[3],zmm0[4],zmm1[5],zmm0[6],zmm1[7]
+; FMA3_512-NEXT:    vfmsubadd213pd  %zmm2, %zmm1, %zmm0
 ; FMA3_512-NEXT:    retq
 ;
 ; FMA4-LABEL: mul_subadd_pd512:
 ; FMA4:       # BB#0: # %entry
-; FMA4-NEXT:    vmulpd %ymm2, %ymm0, %ymm0
-; FMA4-NEXT:    vmulpd %ymm3, %ymm1, %ymm1
-; FMA4-NEXT:    vsubpd %ymm5, %ymm1, %ymm2
-; FMA4-NEXT:    vsubpd %ymm4, %ymm0, %ymm3
-; FMA4-NEXT:    vaddpd %ymm5, %ymm1, %ymm1
-; FMA4-NEXT:    vblendpd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3]
-; FMA4-NEXT:    vaddpd %ymm4, %ymm0, %ymm0
-; FMA4-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2],ymm3[3]
+; FMA4-NEXT:    vfmsubaddpd %ymm4, %ymm2, %ymm0, %ymm0
+; FMA4-NEXT:    vfmsubaddpd %ymm5, %ymm3, %ymm1, %ymm1
 ; FMA4-NEXT:    retq
 entry:
   %AB = fmul <8 x double> %A, %B
   %Sub = fsub <8 x double> %AB, %C
   %Add = fadd <8 x double> %AB, %C
   %subadd = shufflevector <8 x double> %Add, <8 x double> %Sub, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
   ret <8 x double> %subadd
 }
 
 define <16 x float> @mul_subadd_ps512(<16 x float> %A, <16 x float> %B, <16 x float> %C) #0 {
 ; FMA3_256-LABEL: mul_subadd_ps512:
 ; FMA3_256:       # BB#0: # %entry
-; FMA3_256-NEXT:    vmulps %ymm2, %ymm0, %ymm0
-; FMA3_256-NEXT:    vmulps %ymm3, %ymm1, %ymm1
-; FMA3_256-NEXT:    vsubps %ymm5, %ymm1, %ymm2
-; FMA3_256-NEXT:    vsubps %ymm4, %ymm0, %ymm3
-; FMA3_256-NEXT:    vaddps %ymm5, %ymm1, %ymm1
-; FMA3_256-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3],ymm1[4],ymm2[5],ymm1[6],ymm2[7]
-; FMA3_256-NEXT:    vaddps %ymm4, %ymm0, %ymm0
-; FMA3_256-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2],ymm3[3],ymm0[4],ymm3[5],ymm0[6],ymm3[7]
+; FMA3_256-NEXT:    vfmsubadd213ps  %ymm4, %ymm2, %ymm0
+; FMA3_256-NEXT:    vfmsubadd213ps  %ymm5, %ymm3, %ymm1
 ; FMA3_256-NEXT:    retq
 ;
 ; FMA3_512-LABEL: mul_subadd_ps512:
 ; FMA3_512:       # BB#0: # %entry
-; FMA3_512-NEXT:    vmulps %zmm1, %zmm0, %zmm1
-; FMA3_512-NEXT:    vaddps %zmm2, %zmm1, %zmm0
-; FMA3_512-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; FMA3_512-NEXT:    kmovw %eax, %k1
-; FMA3_512-NEXT:    vsubps %zmm2, %zmm1, %zmm0 {%k1}
+; FMA3_512-NEXT:    vfmsubadd213ps  %zmm2, %zmm1, %zmm0
 ; FMA3_512-NEXT:    retq
 ;
 ; FMA4-LABEL: mul_subadd_ps512:
 ; FMA4:       # BB#0: # %entry
-; FMA4-NEXT:    vmulps %ymm2, %ymm0, %ymm0
-; FMA4-NEXT:    vmulps %ymm3, %ymm1, %ymm1
-; FMA4-NEXT:    vsubps %ymm5, %ymm1, %ymm2
-; FMA4-NEXT:    vsubps %ymm4, %ymm0, %ymm3
-; FMA4-NEXT:    vaddps %ymm5, %ymm1, %ymm1
-; FMA4-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3],ymm1[4],ymm2[5],ymm1[6],ymm2[7]
-; FMA4-NEXT:    vaddps %ymm4, %ymm0, %ymm0
-; FMA4-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2],ymm3[3],ymm0[4],ymm3[5],ymm0[6],ymm3[7]
+; FMA4-NEXT:    vfmsubaddps %ymm4, %ymm2, %ymm0, %ymm0
+; FMA4-NEXT:    vfmsubaddps %ymm5, %ymm3, %ymm1, %ymm1
 ; FMA4-NEXT:    retq
 entry:
   %AB = fmul <16 x float> %A, %B
   %Sub = fsub <16 x float> %AB, %C
   %Add = fadd <16 x float> %AB, %C
   %subadd = shufflevector <16 x float> %Add, <16 x float> %Sub, <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
   ret <16 x float> %subadd
 }
 
 attributes #0 = { nounwind "unsafe-fp-math"="true" }