[InstCombine] Expand contractable fmuladd into fmul + fadd

[foad]

Summary

The intention of fmuladd is that it represents an fmul and an fadd that
may be fused with each other but may not be fused with any other
instructions. If the fmuladd is marked as contractable then this
restriction disappears, so we should canonicalize it to separate
(contractable) fmul and fadd instructions, which are generally better
supported by InstCombine and other optimization passes.

[nikic]

I don't think I'm sufficiently familiar with FP semantics to review this...

[foad]

Thread about the intention of fmuladd: https://lists.llvm.org/pipermail/llvm-commits/Week-of-Mon-20120528/143601.html

[arsenm]

Checking fast is way too much. At most it might also need nsz, but I'm pretty sure contract is sufficient

llvm/test/Transforms/InstCombine/fma.ll

230

Could use tests that other flags are preserved

[fhahn]

Is there a test that shows the actual benefit of this, e..g due to further combines? One potential issue is that (parts of) the cost model won't consider fadd/fmul pairs as single instruction, so. there may be negative fallout in things like SLPVectorizer.

[arsenm]

In D152166#4455742, @fhahn wrote:

Is there a test that shows the actual benefit of this, e..g due to further combines? One potential issue is that (parts of) the cost model won't consider fadd/fmul pairs as single instruction, so. there may be negative fallout in things like SLPVectorizer.

This is a question for whether we want the original combine here or to remove it. If we're going to do this it at all it should use the minimal required flags

[foad]

Rebase.

[foad]

In D152166#4455742, @fhahn wrote:

Is there a test that shows the actual benefit of this, e..g due to further combines? One potential issue is that (parts of) the cost model won't consider fadd/fmul pairs as single instruction, so. there may be negative fallout in things like SLPVectorizer.

I added test/CodeGen/AMDGPU/fmuladd-to-fma.ll which shows one case where we get better optimization. I didn't look into exactly why that happens in this case.

llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp

if (ElementCount::isKnownGT(NegatedCount, RetCount)) {
       NewArgs[NegatedOpArg] = OpNotNeg;
       Instruction *NewMul =
           Builder.CreateIntrinsic(II->getType(), IID, NewArgs, II);
       return replaceInstUsesWith(*II, Builder.CreateFNegFMF(NewMul, II));
     }
     break;
   }
   case Intrinsic::fmuladd: {
-    // Canonicalize fast fmuladd to the separate fmul + fadd.
-    if (II->isFast()) {
+    // Canonicalize contractable fmuladd to the separate fmul + fadd.
+    if (II->hasAllowContract()) {
       BuilderTy::FastMathFlagGuard Guard(Builder);
       Builder.setFastMathFlags(II->getFastMathFlags());
       Value *Mul = Builder.CreateFMul(II->getArgOperand(0),
                                       II->getArgOperand(1));
       Value *Add = Builder.CreateFAdd(Mul, II->getArgOperand(2));
       Add->takeName(II);
       return replaceInstUsesWith(*II, Add);
     }

llvm/test/CodeGen/AMDGPU/fmuladd-to-fma.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt < %s -passes=instcombine | llc -march=amdgcn -mcpu=gfx1030 | FileCheck %s
 
 define float @f(float %arg) {
 ; CHECK-LABEL: f:
 ; CHECK:       ; %bb.0: ; %bb
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_waitcnt_vscnt null, 0x0
-; CHECK-NEXT:    v_fma_f32 v0, v0, v0, 1.0
-; CHECK-NEXT:    v_add_f32_e32 v0, -1.0, v0
+; CHECK-NEXT:    v_mul_f32_e32 v0, v0, v0
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %a = call contract float @llvm.fmuladd.f32(float %arg, float %arg, float 1.0)
   %b = fadd reassoc nsz float %a, -1.0
   ret float %b
 }
 
 declare float @llvm.fmuladd.f32(float, float, float)

llvm/test/Transforms/InstCombine/fma.ll

;
   %x.fneg = fneg float %x
   %y.fneg = fneg float %y
   %fmuladd = call float @llvm.fmuladd.f32(float %x.fneg, float %y.fneg, float %z)
   ret float %fmuladd
 }
 
 define float @fmuladd_fneg_x_fneg_y_contract(float %x, float %y, float %z) {
 ; CHECK-LABEL: @fmuladd_fneg_x_fneg_y_contract(
-; CHECK-NEXT:    [[FMULADD:%.*]] = call contract float @llvm.fmuladd.f32(float [[X:%.*]], float [[Y:%.*]], float [[Z:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul contract float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[FMULADD:%.*]] = fadd contract float [[TMP1]], [[Z:%.*]]
 ; CHECK-NEXT:    ret float [[FMULADD]]
 ;
   %x.fneg = fsub float -0.0, %x
   %y.fneg = fsub float -0.0, %y
   %fmuladd = call contract float @llvm.fmuladd.f32(float %x.fneg, float %y.fneg, float %z)
   ret float %fmuladd
 }
 
 define float @fmuladd_fneg_x_fneg_y_fast(float %x, float %y, float %z) {
 ; CHECK-LABEL: @fmuladd_fneg_x_fneg_y_fast(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    [[FMULADD:%.*]] = fadd fast float [[TMP1]], [[Z:%.*]]
 ; CHECK-NEXT:    ret float [[FMULADD]]
 ;
   %x.fneg = fsub float -0.0, %x
   %y.fneg = fsub float -0.0, %y
   %fmuladd = call fast float @llvm.fmuladd.f32(float %x.fneg, float %y.fneg, float %z)
   ret float %fmuladd
 }
 
 define float @fmuladd_unary_fneg_x_unary_fneg_y_contract(float %x, float %y, float %z) {
 ; CHECK-LABEL: @fmuladd_unary_fneg_x_unary_fneg_y_contract(
-; CHECK-NEXT:    [[FMULADD:%.*]] = call contract float @llvm.fmuladd.f32(float [[X:%.*]], float [[Y:%.*]], float [[Z:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul contract float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[FMULADD:%.*]] = fadd contract float [[TMP1]], [[Z:%.*]]
 ; CHECK-NEXT:    ret float [[FMULADD]]
 ;
   %x.fneg = fneg float %x
   %y.fneg = fneg float %y
   %fmuladd = call contract float @llvm.fmuladd.f32(float %x.fneg, float %y.fneg, float %z)
   ret float %fmuladd
 }
 
 define float @fmuladd_unary_fneg_x_unary_fneg_y_fast(float %x, float %y, float %z) {
 ; CHECK-LABEL: @fmuladd_unary_fneg_x_unary_fneg_y_fast(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    [[FMULADD:%.*]] = fadd fast float [[TMP1]], [[Z:%.*]]
 ; CHECK-NEXT:    ret float [[FMULADD]]
 ;
   %x.fneg = fneg float %x
   %y.fneg = fneg float %y

 ;
   %x.fabs = call float @llvm.fabs.f32(float %x)
   %fmuladd = call float @llvm.fmuladd.f32(float %x.fabs, float %x.fabs, float %z)
   ret float %fmuladd
 }
 
 define float @fmuladd_fabs_x_fabs_x_contract(float %x, float %z) {
 ; CHECK-LABEL: @fmuladd_fabs_x_fabs_x_contract(
-; CHECK-NEXT:    [[FMULADD:%.*]] = call contract float @llvm.fmuladd.f32(float [[X:%.*]], float [[X]], float [[Z:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul contract float [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[FMULADD:%.*]] = fadd contract float [[TMP1]], [[Z:%.*]]
 ; CHECK-NEXT:    ret float [[FMULADD]]
 ;
   %x.fabs = call float @llvm.fabs.f32(float %x)
   %fmuladd = call contract float @llvm.fmuladd.f32(float %x.fabs, float %x.fabs, float %z)
   ret float %fmuladd
 }
 
 define float @fmuladd_fabs_x_fabs_x_fast(float %x, float %z) {

 ; CHECK-NEXT:    ret float [[FMA]]
 ;
   %fma = call fast float @llvm.fma.f32(float 4.0, float %y, float %z)
   ret float %fma
 }
 
 define float @fmuladd_k_y_z_contract(float %y, float %z) {
 ; CHECK-LABEL: @fmuladd_k_y_z_contract(
-; CHECK-NEXT:    [[FMULADD:%.*]] = call contract float @llvm.fmuladd.f32(float [[Y:%.*]], float 4.000000e+00, float [[Z:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul contract float [[Y:%.*]], 4.000000e+00
+; CHECK-NEXT:    [[FMULADD:%.*]] = fadd contract float [[TMP1]], [[Z:%.*]]
 ; CHECK-NEXT:    ret float [[FMULADD]]
 ;
   %fmuladd = call contract float @llvm.fmuladd.f32(float 4.0, float %y, float %z)
   ret float %fmuladd
 }
 
 define float @fmuladd_k_y_z_fast(float %y, float %z) {
 ; CHECK-LABEL: @fmuladd_k_y_z_fast(

llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-dominance.ll

 ; CHECK-NEXT:    [[TMP3:%.*]] = phi ptr [ [[B]], [[ENTRY:%.*]] ], [ [[B]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
 ; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <1 x double>, ptr [[A:%.*]], align 8
 ; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <1 x double>, ptr [[TMP3]], align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = fmul contract <1 x double> [[COL_LOAD]], [[COL_LOAD1]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, ptr [[A]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <1 x double>, ptr [[TMP5]], align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
 ; CHECK-NEXT:    [[COL_LOAD3:%.*]] = load <1 x double>, ptr [[TMP6]], align 8
-; CHECK-NEXT:    [[TMP7:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD2]], <1 x double> [[COL_LOAD3]], <1 x double> [[TMP4]])
-; CHECK-NEXT:    store <1 x double> [[TMP7]], ptr [[C]], align 8
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr double, ptr [[A]], i64 1
-; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <1 x double>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul contract <1 x double> [[COL_LOAD2]], [[COL_LOAD3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd contract <1 x double> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    store <1 x double> [[TMP8]], ptr [[C]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr double, ptr [[A]], i64 1
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <1 x double>, ptr [[TMP9]], align 8
 ; CHECK-NEXT:    [[COL_LOAD9:%.*]] = load <1 x double>, ptr [[TMP3]], align 8
-; CHECK-NEXT:    [[TMP9:%.*]] = fmul contract <1 x double> [[COL_LOAD8]], [[COL_LOAD9]]
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr double, ptr [[A]], i64 3
-; CHECK-NEXT:    [[COL_LOAD13:%.*]] = load <1 x double>, ptr [[TMP10]], align 8
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
-; CHECK-NEXT:    [[COL_LOAD14:%.*]] = load <1 x double>, ptr [[TMP11]], align 8
-; CHECK-NEXT:    [[TMP12:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD13]], <1 x double> [[COL_LOAD14]], <1 x double> [[TMP9]])
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr double, ptr [[C]], i64 1
-; CHECK-NEXT:    store <1 x double> [[TMP12]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul contract <1 x double> [[COL_LOAD8]], [[COL_LOAD9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT:    [[COL_LOAD13:%.*]] = load <1 x double>, ptr [[TMP11]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
+; CHECK-NEXT:    [[COL_LOAD14:%.*]] = load <1 x double>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = fmul contract <1 x double> [[COL_LOAD13]], [[COL_LOAD14]]
+; CHECK-NEXT:    [[TMP14:%.*]] = fadd contract <1 x double> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr double, ptr [[C]], i64 1
+; CHECK-NEXT:    store <1 x double> [[TMP14]], ptr [[TMP15]], align 8
 ; CHECK-NEXT:    [[COL_LOAD19:%.*]] = load <1 x double>, ptr [[A]], align 8
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
-; CHECK-NEXT:    [[COL_LOAD20:%.*]] = load <1 x double>, ptr [[TMP14]], align 8
-; CHECK-NEXT:    [[TMP15:%.*]] = fmul contract <1 x double> [[COL_LOAD19]], [[COL_LOAD20]]
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr double, ptr [[A]], i64 2
-; CHECK-NEXT:    [[COL_LOAD24:%.*]] = load <1 x double>, ptr [[TMP16]], align 8
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
-; CHECK-NEXT:    [[COL_LOAD25:%.*]] = load <1 x double>, ptr [[TMP17]], align 8
-; CHECK-NEXT:    [[TMP18:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[COL_LOAD25]], <1 x double> [[TMP15]])
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr double, ptr [[C]], i64 2
-; CHECK-NEXT:    store <1 x double> [[TMP18]], ptr [[TMP19]], align 8
-; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr double, ptr [[A]], i64 1
-; CHECK-NEXT:    [[COL_LOAD30:%.*]] = load <1 x double>, ptr [[TMP20]], align 8
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
-; CHECK-NEXT:    [[COL_LOAD31:%.*]] = load <1 x double>, ptr [[TMP21]], align 8
-; CHECK-NEXT:    [[TMP22:%.*]] = fmul contract <1 x double> [[COL_LOAD30]], [[COL_LOAD31]]
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr double, ptr [[A]], i64 3
-; CHECK-NEXT:    [[COL_LOAD35:%.*]] = load <1 x double>, ptr [[TMP23]], align 8
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
-; CHECK-NEXT:    [[COL_LOAD36:%.*]] = load <1 x double>, ptr [[TMP24]], align 8
-; CHECK-NEXT:    [[TMP25:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD35]], <1 x double> [[COL_LOAD36]], <1 x double> [[TMP22]])
-; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr double, ptr [[C]], i64 3
-; CHECK-NEXT:    store <1 x double> [[TMP25]], ptr [[TMP26]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT:    [[COL_LOAD20:%.*]] = load <1 x double>, ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[TMP17:%.*]] = fmul contract <1 x double> [[COL_LOAD19]], [[COL_LOAD20]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT:    [[COL_LOAD24:%.*]] = load <1 x double>, ptr [[TMP18]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
+; CHECK-NEXT:    [[COL_LOAD25:%.*]] = load <1 x double>, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP20:%.*]] = fmul contract <1 x double> [[COL_LOAD24]], [[COL_LOAD25]]
+; CHECK-NEXT:    [[TMP21:%.*]] = fadd contract <1 x double> [[TMP20]], [[TMP17]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr double, ptr [[C]], i64 2
+; CHECK-NEXT:    store <1 x double> [[TMP21]], ptr [[TMP22]], align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr double, ptr [[A]], i64 1
+; CHECK-NEXT:    [[COL_LOAD30:%.*]] = load <1 x double>, ptr [[TMP23]], align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT:    [[COL_LOAD31:%.*]] = load <1 x double>, ptr [[TMP24]], align 8
+; CHECK-NEXT:    [[TMP25:%.*]] = fmul contract <1 x double> [[COL_LOAD30]], [[COL_LOAD31]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT:    [[COL_LOAD35:%.*]] = load <1 x double>, ptr [[TMP26]], align 8
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
+; CHECK-NEXT:    [[COL_LOAD36:%.*]] = load <1 x double>, ptr [[TMP27]], align 8
+; CHECK-NEXT:    [[TMP28:%.*]] = fmul contract <1 x double> [[COL_LOAD35]], [[COL_LOAD36]]
+; CHECK-NEXT:    [[TMP29:%.*]] = fadd contract <1 x double> [[TMP28]], [[TMP25]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr double, ptr [[C]], i64 3
+; CHECK-NEXT:    store <1 x double> [[TMP29]], ptr [[TMP30]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %a = load <4 x double>, ptr %A, align 8
   %b = load <4 x double>, ptr %B, align 8
   %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
   store <4 x double> %c, ptr %C, align 8
   ret void

 ; CHECK-NEXT:    [[TMP3:%.*]] = phi ptr [ [[B]], [[ENTRY:%.*]] ], [ [[B]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
 ; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <1 x double>, ptr [[A:%.*]], align 8
 ; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <1 x double>, ptr [[TMP3]], align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = fmul contract <1 x double> [[COL_LOAD]], [[COL_LOAD1]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, ptr [[A]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <1 x double>, ptr [[TMP5]], align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
 ; CHECK-NEXT:    [[COL_LOAD3:%.*]] = load <1 x double>, ptr [[TMP6]], align 8
-; CHECK-NEXT:    [[TMP7:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD2]], <1 x double> [[COL_LOAD3]], <1 x double> [[TMP4]])
-; CHECK-NEXT:    store <1 x double> [[TMP7]], ptr [[GEP]], align 8
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr double, ptr [[A]], i64 1
-; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <1 x double>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul contract <1 x double> [[COL_LOAD2]], [[COL_LOAD3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd contract <1 x double> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    store <1 x double> [[TMP8]], ptr [[GEP]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr double, ptr [[A]], i64 1
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <1 x double>, ptr [[TMP9]], align 8
 ; CHECK-NEXT:    [[COL_LOAD9:%.*]] = load <1 x double>, ptr [[TMP3]], align 8
-; CHECK-NEXT:    [[TMP9:%.*]] = fmul contract <1 x double> [[COL_LOAD8]], [[COL_LOAD9]]
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr double, ptr [[A]], i64 3
-; CHECK-NEXT:    [[COL_LOAD13:%.*]] = load <1 x double>, ptr [[TMP10]], align 8
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
-; CHECK-NEXT:    [[COL_LOAD14:%.*]] = load <1 x double>, ptr [[TMP11]], align 8
-; CHECK-NEXT:    [[TMP12:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD13]], <1 x double> [[COL_LOAD14]], <1 x double> [[TMP9]])
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr [4 x double], ptr [[C]], i64 2, i64 1
-; CHECK-NEXT:    store <1 x double> [[TMP12]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul contract <1 x double> [[COL_LOAD8]], [[COL_LOAD9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT:    [[COL_LOAD13:%.*]] = load <1 x double>, ptr [[TMP11]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
+; CHECK-NEXT:    [[COL_LOAD14:%.*]] = load <1 x double>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = fmul contract <1 x double> [[COL_LOAD13]], [[COL_LOAD14]]
+; CHECK-NEXT:    [[TMP14:%.*]] = fadd contract <1 x double> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr [4 x double], ptr [[C]], i64 2, i64 1
+; CHECK-NEXT:    store <1 x double> [[TMP14]], ptr [[TMP15]], align 8
 ; CHECK-NEXT:    [[COL_LOAD19:%.*]] = load <1 x double>, ptr [[A]], align 8
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
-; CHECK-NEXT:    [[COL_LOAD20:%.*]] = load <1 x double>, ptr [[TMP14]], align 8
-; CHECK-NEXT:    [[TMP15:%.*]] = fmul contract <1 x double> [[COL_LOAD19]], [[COL_LOAD20]]
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr double, ptr [[A]], i64 2
-; CHECK-NEXT:    [[COL_LOAD24:%.*]] = load <1 x double>, ptr [[TMP16]], align 8
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
-; CHECK-NEXT:    [[COL_LOAD25:%.*]] = load <1 x double>, ptr [[TMP17]], align 8
-; CHECK-NEXT:    [[TMP18:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[COL_LOAD25]], <1 x double> [[TMP15]])
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr [4 x double], ptr [[C]], i64 2, i64 2
-; CHECK-NEXT:    store <1 x double> [[TMP18]], ptr [[TMP19]], align 8
-; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr double, ptr [[A]], i64 1
-; CHECK-NEXT:    [[COL_LOAD30:%.*]] = load <1 x double>, ptr [[TMP20]], align 8
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
-; CHECK-NEXT:    [[COL_LOAD31:%.*]] = load <1 x double>, ptr [[TMP21]], align 8
-; CHECK-NEXT:    [[TMP22:%.*]] = fmul contract <1 x double> [[COL_LOAD30]], [[COL_LOAD31]]
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr double, ptr [[A]], i64 3
-; CHECK-NEXT:    [[COL_LOAD35:%.*]] = load <1 x double>, ptr [[TMP23]], align 8
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
-; CHECK-NEXT:    [[COL_LOAD36:%.*]] = load <1 x double>, ptr [[TMP24]], align 8
-; CHECK-NEXT:    [[TMP25:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD35]], <1 x double> [[COL_LOAD36]], <1 x double> [[TMP22]])
-; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr [4 x double], ptr [[C]], i64 2, i64 3
-; CHECK-NEXT:    store <1 x double> [[TMP25]], ptr [[TMP26]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT:    [[COL_LOAD20:%.*]] = load <1 x double>, ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[TMP17:%.*]] = fmul contract <1 x double> [[COL_LOAD19]], [[COL_LOAD20]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT:    [[COL_LOAD24:%.*]] = load <1 x double>, ptr [[TMP18]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
+; CHECK-NEXT:    [[COL_LOAD25:%.*]] = load <1 x double>, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP20:%.*]] = fmul contract <1 x double> [[COL_LOAD24]], [[COL_LOAD25]]
+; CHECK-NEXT:    [[TMP21:%.*]] = fadd contract <1 x double> [[TMP20]], [[TMP17]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr [4 x double], ptr [[C]], i64 2, i64 2
+; CHECK-NEXT:    store <1 x double> [[TMP21]], ptr [[TMP22]], align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr double, ptr [[A]], i64 1
+; CHECK-NEXT:    [[COL_LOAD30:%.*]] = load <1 x double>, ptr [[TMP23]], align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT:    [[COL_LOAD31:%.*]] = load <1 x double>, ptr [[TMP24]], align 8
+; CHECK-NEXT:    [[TMP25:%.*]] = fmul contract <1 x double> [[COL_LOAD30]], [[COL_LOAD31]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT:    [[COL_LOAD35:%.*]] = load <1 x double>, ptr [[TMP26]], align 8
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
+; CHECK-NEXT:    [[COL_LOAD36:%.*]] = load <1 x double>, ptr [[TMP27]], align 8
+; CHECK-NEXT:    [[TMP28:%.*]] = fmul contract <1 x double> [[COL_LOAD35]], [[COL_LOAD36]]
+; CHECK-NEXT:    [[TMP29:%.*]] = fadd contract <1 x double> [[TMP28]], [[TMP25]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr [4 x double], ptr [[C]], i64 2, i64 3
+; CHECK-NEXT:    store <1 x double> [[TMP29]], ptr [[TMP30]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %a = load <4 x double>, ptr %A, align 8
   %b = load <4 x double>, ptr %B, align 8
   %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
   %gep = getelementptr [4 x double], ptr %C, i32 2
   store <4 x double> %c, ptr %gep, align 8

 ; CHECK-NEXT:    [[TMP3:%.*]] = phi ptr [ [[B]], [[ENTRY:%.*]] ], [ [[B]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
 ; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <1 x double>, ptr [[A:%.*]], align 8
 ; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <1 x double>, ptr [[TMP3]], align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = fmul contract <1 x double> [[COL_LOAD]], [[COL_LOAD1]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, ptr [[A]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <1 x double>, ptr [[TMP5]], align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
 ; CHECK-NEXT:    [[COL_LOAD3:%.*]] = load <1 x double>, ptr [[TMP6]], align 8
-; CHECK-NEXT:    [[TMP7:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD2]], <1 x double> [[COL_LOAD3]], <1 x double> [[TMP4]])
-; CHECK-NEXT:    store <1 x double> [[TMP7]], ptr [[GEP_1]], align 8
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr double, ptr [[A]], i64 1
-; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <1 x double>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul contract <1 x double> [[COL_LOAD2]], [[COL_LOAD3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd contract <1 x double> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    store <1 x double> [[TMP8]], ptr [[GEP_1]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr double, ptr [[A]], i64 1
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <1 x double>, ptr [[TMP9]], align 8
 ; CHECK-NEXT:    [[COL_LOAD9:%.*]] = load <1 x double>, ptr [[TMP3]], align 8
-; CHECK-NEXT:    [[TMP9:%.*]] = fmul contract <1 x double> [[COL_LOAD8]], [[COL_LOAD9]]
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr double, ptr [[A]], i64 3
-; CHECK-NEXT:    [[COL_LOAD13:%.*]] = load <1 x double>, ptr [[TMP10]], align 8
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
-; CHECK-NEXT:    [[COL_LOAD14:%.*]] = load <1 x double>, ptr [[TMP11]], align 8
-; CHECK-NEXT:    [[TMP12:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD13]], <1 x double> [[COL_LOAD14]], <1 x double> [[TMP9]])
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[C]], i64 1352
-; CHECK-NEXT:    store <1 x double> [[TMP12]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul contract <1 x double> [[COL_LOAD8]], [[COL_LOAD9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT:    [[COL_LOAD13:%.*]] = load <1 x double>, ptr [[TMP11]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
+; CHECK-NEXT:    [[COL_LOAD14:%.*]] = load <1 x double>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = fmul contract <1 x double> [[COL_LOAD13]], [[COL_LOAD14]]
+; CHECK-NEXT:    [[TMP14:%.*]] = fadd contract <1 x double> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[C]], i64 1352
+; CHECK-NEXT:    store <1 x double> [[TMP14]], ptr [[TMP15]], align 8
 ; CHECK-NEXT:    [[COL_LOAD19:%.*]] = load <1 x double>, ptr [[A]], align 8
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
-; CHECK-NEXT:    [[COL_LOAD20:%.*]] = load <1 x double>, ptr [[TMP14]], align 8
-; CHECK-NEXT:    [[TMP15:%.*]] = fmul contract <1 x double> [[COL_LOAD19]], [[COL_LOAD20]]
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr double, ptr [[A]], i64 2
-; CHECK-NEXT:    [[COL_LOAD24:%.*]] = load <1 x double>, ptr [[TMP16]], align 8
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
-; CHECK-NEXT:    [[COL_LOAD25:%.*]] = load <1 x double>, ptr [[TMP17]], align 8
-; CHECK-NEXT:    [[TMP18:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[COL_LOAD25]], <1 x double> [[TMP15]])
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[C]], i64 1360
-; CHECK-NEXT:    store <1 x double> [[TMP18]], ptr [[TMP19]], align 8
-; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr double, ptr [[A]], i64 1
-; CHECK-NEXT:    [[COL_LOAD30:%.*]] = load <1 x double>, ptr [[TMP20]], align 8
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
-; CHECK-NEXT:    [[COL_LOAD31:%.*]] = load <1 x double>, ptr [[TMP21]], align 8
-; CHECK-NEXT:    [[TMP22:%.*]] = fmul contract <1 x double> [[COL_LOAD30]], [[COL_LOAD31]]
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr double, ptr [[A]], i64 3
-; CHECK-NEXT:    [[COL_LOAD35:%.*]] = load <1 x double>, ptr [[TMP23]], align 8
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
-; CHECK-NEXT:    [[COL_LOAD36:%.*]] = load <1 x double>, ptr [[TMP24]], align 8
-; CHECK-NEXT:    [[TMP25:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD35]], <1 x double> [[COL_LOAD36]], <1 x double> [[TMP22]])
-; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[C]], i64 1368
-; CHECK-NEXT:    store <1 x double> [[TMP25]], ptr [[TMP26]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT:    [[COL_LOAD20:%.*]] = load <1 x double>, ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[TMP17:%.*]] = fmul contract <1 x double> [[COL_LOAD19]], [[COL_LOAD20]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT:    [[COL_LOAD24:%.*]] = load <1 x double>, ptr [[TMP18]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
+; CHECK-NEXT:    [[COL_LOAD25:%.*]] = load <1 x double>, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP20:%.*]] = fmul contract <1 x double> [[COL_LOAD24]], [[COL_LOAD25]]
+; CHECK-NEXT:    [[TMP21:%.*]] = fadd contract <1 x double> [[TMP20]], [[TMP17]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[C]], i64 1360
+; CHECK-NEXT:    store <1 x double> [[TMP21]], ptr [[TMP22]], align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr double, ptr [[A]], i64 1
+; CHECK-NEXT:    [[COL_LOAD30:%.*]] = load <1 x double>, ptr [[TMP23]], align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT:    [[COL_LOAD31:%.*]] = load <1 x double>, ptr [[TMP24]], align 8
+; CHECK-NEXT:    [[TMP25:%.*]] = fmul contract <1 x double> [[COL_LOAD30]], [[COL_LOAD31]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT:    [[COL_LOAD35:%.*]] = load <1 x double>, ptr [[TMP26]], align 8
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
+; CHECK-NEXT:    [[COL_LOAD36:%.*]] = load <1 x double>, ptr [[TMP27]], align 8
+; CHECK-NEXT:    [[TMP28:%.*]] = fmul contract <1 x double> [[COL_LOAD35]], [[COL_LOAD36]]
+; CHECK-NEXT:    [[TMP29:%.*]] = fadd contract <1 x double> [[TMP28]], [[TMP25]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[C]], i64 1368
+; CHECK-NEXT:    store <1 x double> [[TMP29]], ptr [[TMP30]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %a = load <4 x double>, ptr %A, align 8
   %b = load <4 x double>, ptr %B, align 8
   %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
   %off.0 = add i32 10, 10
   %off.1 = add i32 %off.0, 2
   %off.2 = add i32 %off.0, %off.1
   %gep.1 = getelementptr <4 x double>, ptr %C, i32 %off.2
   store <4 x double> %c, ptr %gep.1, align 8
   ret void
 }
 
 define void @multiply_dont_hoist_phi(ptr noalias %A, ptr %B, ptr %C) {
 ; CHECK-LABEL: @multiply_dont_hoist_phi(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[NEXT:%.*]]
 ; CHECK:       next:
 ; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[A:%.*]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
 ; CHECK-NEXT:    [[VEC_GEP3:%.*]] = getelementptr double, ptr [[B:%.*]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD1]], [[SPLAT_SPLAT13]]
 ; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
-; CHECK-NEXT:    [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd contract <2 x double> [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[B]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul contract <2 x double> [[COL_LOAD1]], [[SPLAT_SPLAT7]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd contract <2 x double> [[TMP3]], [[TMP4]]
 ; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr <4 x double>, ptr [[C:%.*]], i64 26
-; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[GEP_1]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP5]], ptr [[GEP_1]], align 8
 ; CHECK-NEXT:    [[VEC_GEP14:%.*]] = getelementptr i8, ptr [[C]], i64 848
-; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[VEC_GEP14]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP2]], ptr [[VEC_GEP14]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %a = load <4 x double>, ptr %A, align 8
   %b = load <4 x double>, ptr %B, align 8
   %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
   br label %next
 

 ; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
 ; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[B:%.*]], align 8
 ; CHECK-NEXT:    [[VEC_GEP3:%.*]] = getelementptr double, ptr [[B]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul contract <2 x double> [[COL_LOAD1]], [[SPLAT_SPLAT7]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd contract <2 x double> [[TMP1]], [[TMP0]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul contract <2 x double> [[COL_LOAD1]], [[SPLAT_SPLAT13]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd contract <2 x double> [[TMP4]], [[TMP3]]
 ; CHECK-NEXT:    [[C:%.*]] = load ptr, ptr [[C_PTR:%.*]], align 8
-; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[C]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP2]], ptr [[C]], align 8
 ; CHECK-NEXT:    [[VEC_GEP14:%.*]] = getelementptr double, ptr [[C]], i64 2
-; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[VEC_GEP14]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP5]], ptr [[VEC_GEP14]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %a = load <4 x double>, ptr %A, align 8
   %b = load <4 x double>, ptr %B, align 8
   %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
   %C = load ptr, ptr %C.ptr
   store <4 x double> %c, ptr %C, align 8
   ret void
 }
 
 ; The address load may alias, so avoid moving it for now.
 define void @multiply_dont_hoist_load(ptr noalias %A, ptr %B, ptr %C.ptr) {
 ; CHECK-LABEL: @multiply_dont_hoist_load(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
 ; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
 ; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[B:%.*]], align 8
 ; CHECK-NEXT:    [[VEC_GEP3:%.*]] = getelementptr double, ptr [[B]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul contract <2 x double> [[COL_LOAD1]], [[SPLAT_SPLAT7]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd contract <2 x double> [[TMP1]], [[TMP0]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul contract <2 x double> [[COL_LOAD1]], [[SPLAT_SPLAT13]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd contract <2 x double> [[TMP4]], [[TMP3]]
 ; CHECK-NEXT:    [[C:%.*]] = load ptr, ptr [[C_PTR:%.*]], align 8
-; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[C]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP2]], ptr [[C]], align 8
 ; CHECK-NEXT:    [[VEC_GEP14:%.*]] = getelementptr double, ptr [[C]], i64 2
-; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[VEC_GEP14]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP5]], ptr [[VEC_GEP14]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %a = load <4 x double>, ptr %A, align 8
   %b = load <4 x double>, ptr %B, align 8
   %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
   %C = load ptr, ptr %C.ptr
   store <4 x double> %c, ptr %C, align 8
   ret void
 }
 
 ; The call to @get_adress may clobber memory, avoid moving it for now.
 define void @multiply_dont_hoist_call(ptr noalias %A, ptr %B) {
 ; CHECK-LABEL: @multiply_dont_hoist_call(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
 ; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
 ; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[B:%.*]], align 8
 ; CHECK-NEXT:    [[VEC_GEP3:%.*]] = getelementptr double, ptr [[B]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul contract <2 x double> [[COL_LOAD1]], [[SPLAT_SPLAT7]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd contract <2 x double> [[TMP1]], [[TMP0]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul contract <2 x double> [[COL_LOAD1]], [[SPLAT_SPLAT13]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd contract <2 x double> [[TMP4]], [[TMP3]]
 ; CHECK-NEXT:    [[C:%.*]] = call ptr @get_address()
-; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[C]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP2]], ptr [[C]], align 8
 ; CHECK-NEXT:    [[VEC_GEP14:%.*]] = getelementptr double, ptr [[C]], i64 2
-; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[VEC_GEP14]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP5]], ptr [[VEC_GEP14]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %a = load <4 x double>, ptr %A, align 8
   %b = load <4 x double>, ptr %B, align 8
   %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
   %C = call ptr @get_address()
   store <4 x double> %c, ptr %C, align 8
   ret void
 }
 
 declare ptr @get_address()
 
 
 declare <4 x double> @llvm.matrix.multiply(<4 x double>, <4 x double>, i32, i32, i32)

llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll

 ; CHECK-NEXT:    br label [[ROWS_HEADER:%.*]]
 ; CHECK:       rows.header:
 ; CHECK-NEXT:    [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ]
 ; CHECK-NEXT:    br label [[ROWS_BODY:%.*]]
 ; CHECK:       rows.body:
 ; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
 ; CHECK:       inner.header:
 ; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
-; CHECK-NEXT:    [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP7:%.*]], [[INNER_LATCH]] ]
-; CHECK-NEXT:    [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP9:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP9:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP13:%.*]], [[INNER_LATCH]] ]
 ; CHECK-NEXT:    br label [[INNER_BODY:%.*]]
 ; CHECK:       inner.body:
 ; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[INNER_IV]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], [[ROWS_IV]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, ptr [[TMP2]], align 8
 ; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[TMP2]], i64 4
 ; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[COLS_IV]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], [[INNER_IV]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[TMP5]], align 8
 ; CHECK-NEXT:    [[VEC_GEP3:%.*]] = getelementptr double, ptr [[TMP5]], i64 4
 ; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[RESULT_VEC_0]])
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd contract <2 x double> [[TMP6]], [[RESULT_VEC_0]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP7]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT8]], <2 x double> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul contract <2 x double> [[COL_LOAD1]], [[SPLAT_SPLAT8]]
+; CHECK-NEXT:    [[TMP9]] = fadd contract <2 x double> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP8:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[RESULT_VEC_1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT12]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd contract <2 x double> [[TMP10]], [[RESULT_VEC_1]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP9]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT15]], <2 x double> [[TMP8]])
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul contract <2 x double> [[COL_LOAD1]], [[SPLAT_SPLAT15]]
+; CHECK-NEXT:    [[TMP13]] = fadd contract <2 x double> [[TMP12]], [[TMP11]]
 ; CHECK-NEXT:    br label [[INNER_LATCH]]
 ; CHECK:       inner.latch:
 ; CHECK-NEXT:    [[INNER_STEP]] = add i64 [[INNER_IV]], 2
 ; CHECK-NEXT:    [[INNER_COND_NOT:%.*]] = icmp eq i64 [[INNER_STEP]], 4
 ; CHECK-NEXT:    br i1 [[INNER_COND_NOT]], label [[ROWS_LATCH]], label [[INNER_HEADER]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       rows.latch:
 ; CHECK-NEXT:    [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
 ; CHECK-NEXT:    [[ROWS_COND_NOT:%.*]] = icmp eq i64 [[ROWS_STEP]], 4
-; CHECK-NEXT:    [[TMP10:%.*]] = shl i64 [[COLS_IV]], 2
-; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], [[ROWS_IV]]
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr double, ptr [[C:%.*]], i64 [[TMP11]]
-; CHECK-NEXT:    store <2 x double> [[TMP7]], ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[VEC_GEP16:%.*]] = getelementptr double, ptr [[TMP12]], i64 4
-; CHECK-NEXT:    store <2 x double> [[TMP9]], ptr [[VEC_GEP16]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = shl i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP14]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr double, ptr [[C:%.*]], i64 [[TMP15]]
+; CHECK-NEXT:    store <2 x double> [[TMP9]], ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[VEC_GEP16:%.*]] = getelementptr double, ptr [[TMP16]], i64 4
+; CHECK-NEXT:    store <2 x double> [[TMP13]], ptr [[VEC_GEP16]], align 8
 ; CHECK-NEXT:    br i1 [[ROWS_COND_NOT]], label [[COLS_LATCH]], label [[ROWS_HEADER]]
 ; CHECK:       cols.latch:
 ; CHECK-NEXT:    [[COLS_STEP]] = add i64 [[COLS_IV]], 2
 ; CHECK-NEXT:    [[COLS_COND_NOT:%.*]] = icmp eq i64 [[COLS_STEP]], 4
 ; CHECK-NEXT:    br i1 [[COLS_COND_NOT]], label [[CONTINUE:%.*]], label [[COLS_HEADER]]
 ; CHECK:       continue:
 ; CHECK-NEXT:    ret void
 ;

 ; CHECK-NEXT:    br label [[ROWS_HEADER:%.*]]
 ; CHECK:       rows.header:
 ; CHECK-NEXT:    [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ]
 ; CHECK-NEXT:    br label [[ROWS_BODY:%.*]]
 ; CHECK:       rows.body:
 ; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
 ; CHECK:       inner.header:
 ; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
-; CHECK-NEXT:    [[RESULT_VEC_0:%.*]] = phi <2 x float> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP15:%.*]], [[INNER_LATCH]] ]
-; CHECK-NEXT:    [[RESULT_VEC_1:%.*]] = phi <2 x float> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    [[RESULT_VEC_0:%.*]] = phi <2 x float> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    [[RESULT_VEC_1:%.*]] = phi <2 x float> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP21:%.*]], [[INNER_LATCH]] ]
 ; CHECK-NEXT:    br label [[INNER_BODY:%.*]]
 ; CHECK:       inner.body:
 ; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[INNER_IV]], 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP8]], [[ROWS_IV]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr float, ptr [[TMP3]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x float>, ptr [[TMP10]], align 4
 ; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr float, ptr [[TMP10]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 4
 ; CHECK-NEXT:    [[TMP11:%.*]] = shl i64 [[COLS_IV]], 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP11]], [[INNER_IV]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr float, ptr [[TMP7]], i64 [[TMP12]]
 ; CHECK-NEXT:    [[COL_LOAD9:%.*]] = load <2 x float>, ptr [[TMP13]], align 4
 ; CHECK-NEXT:    [[VEC_GEP10:%.*]] = getelementptr float, ptr [[TMP13]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD11:%.*]] = load <2 x float>, ptr [[VEC_GEP10]], align 4
 ; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x float> [[COL_LOAD9]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP14:%.*]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD]], <2 x float> [[SPLAT_SPLAT]], <2 x float> [[RESULT_VEC_0]])
+; CHECK-NEXT:    [[TMP14:%.*]] = fmul contract <2 x float> [[COL_LOAD]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[TMP15:%.*]] = fadd contract <2 x float> [[TMP14]], [[RESULT_VEC_0]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x float> [[COL_LOAD9]], <2 x float> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP15]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD8]], <2 x float> [[SPLAT_SPLAT15]], <2 x float> [[TMP14]])
+; CHECK-NEXT:    [[TMP16:%.*]] = fmul contract <2 x float> [[COL_LOAD8]], [[SPLAT_SPLAT15]]
+; CHECK-NEXT:    [[TMP17]] = fadd contract <2 x float> [[TMP16]], [[TMP15]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x float> [[COL_LOAD11]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP16:%.*]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD]], <2 x float> [[SPLAT_SPLAT19]], <2 x float> [[RESULT_VEC_1]])
+; CHECK-NEXT:    [[TMP18:%.*]] = fmul contract <2 x float> [[COL_LOAD]], [[SPLAT_SPLAT19]]
+; CHECK-NEXT:    [[TMP19:%.*]] = fadd contract <2 x float> [[TMP18]], [[RESULT_VEC_1]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT22:%.*]] = shufflevector <2 x float> [[COL_LOAD11]], <2 x float> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP17]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD8]], <2 x float> [[SPLAT_SPLAT22]], <2 x float> [[TMP16]])
+; CHECK-NEXT:    [[TMP20:%.*]] = fmul contract <2 x float> [[COL_LOAD8]], [[SPLAT_SPLAT22]]
+; CHECK-NEXT:    [[TMP21]] = fadd contract <2 x float> [[TMP20]], [[TMP19]]
 ; CHECK-NEXT:    br label [[INNER_LATCH]]
 ; CHECK:       inner.latch:
 ; CHECK-NEXT:    [[INNER_STEP]] = add i64 [[INNER_IV]], 2
 ; CHECK-NEXT:    [[INNER_COND_NOT:%.*]] = icmp eq i64 [[INNER_IV]], 0
 ; CHECK-NEXT:    br i1 [[INNER_COND_NOT]], label [[ROWS_LATCH]], label [[INNER_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       rows.latch:
 ; CHECK-NEXT:    [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
 ; CHECK-NEXT:    [[ROWS_COND_NOT:%.*]] = icmp eq i64 [[ROWS_IV]], 0
-; CHECK-NEXT:    [[TMP18:%.*]] = shl i64 [[COLS_IV]], 1
-; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[TMP18]], [[ROWS_IV]]
-; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr float, ptr [[C]], i64 [[TMP19]]
-; CHECK-NEXT:    store <2 x float> [[TMP15]], ptr [[TMP20]], align 8
-; CHECK-NEXT:    [[VEC_GEP23:%.*]] = getelementptr float, ptr [[TMP20]], i64 2
-; CHECK-NEXT:    store <2 x float> [[TMP17]], ptr [[VEC_GEP23]], align 8
+; CHECK-NEXT:    [[TMP22:%.*]] = shl i64 [[COLS_IV]], 1
+; CHECK-NEXT:    [[TMP23:%.*]] = add i64 [[TMP22]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr float, ptr [[C]], i64 [[TMP23]]
+; CHECK-NEXT:    store <2 x float> [[TMP17]], ptr [[TMP24]], align 8
+; CHECK-NEXT:    [[VEC_GEP23:%.*]] = getelementptr float, ptr [[TMP24]], i64 2
+; CHECK-NEXT:    store <2 x float> [[TMP21]], ptr [[VEC_GEP23]], align 8
 ; CHECK-NEXT:    br i1 [[ROWS_COND_NOT]], label [[COLS_LATCH]], label [[ROWS_HEADER]]
 ; CHECK:       cols.latch:
 ; CHECK-NEXT:    [[COLS_STEP]] = add i64 [[COLS_IV]], 2
 ; CHECK-NEXT:    [[COLS_COND_NOT:%.*]] = icmp eq i64 [[COLS_IV]], 0
 ; CHECK-NEXT:    br i1 [[COLS_COND_NOT]], label [[CONTINUE:%.*]], label [[COLS_HEADER]]
 ; CHECK:       continue:
 ; CHECK-NEXT:    ret void
 ;

llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll

 ; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
 ; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
 ; CHECK-NEXT:    [[COL_LOAD9:%.*]] = load <2 x double>, ptr [[TMP7]], align 8
 ; CHECK-NEXT:    [[VEC_GEP10:%.*]] = getelementptr double, ptr [[TMP7]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD11:%.*]] = load <2 x double>, ptr [[VEC_GEP10]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD9]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT14:%.*]] = shufflevector <2 x double> [[COL_LOAD9]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT14]], <2 x double> [[TMP8]])
+; CHECK-NEXT:    [[TMP9:%.*]] = fmul contract <2 x double> [[COL_LOAD8]], [[SPLAT_SPLAT14]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fadd contract <2 x double> [[TMP9]], [[TMP8]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT17:%.*]] = shufflevector <2 x double> [[COL_LOAD11]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT17]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT17]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT20:%.*]] = shufflevector <2 x double> [[COL_LOAD11]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP11:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT20]], <2 x double> [[TMP10]])
-; CHECK-NEXT:    store <2 x double> [[TMP9]], ptr [[C]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul contract <2 x double> [[COL_LOAD8]], [[SPLAT_SPLAT20]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fadd contract <2 x double> [[TMP12]], [[TMP11]]
+; CHECK-NEXT:    store <2 x double> [[TMP10]], ptr [[C]], align 8
 ; CHECK-NEXT:    [[VEC_GEP21:%.*]] = getelementptr double, ptr [[C]], i64 3
-; CHECK-NEXT:    store <2 x double> [[TMP11]], ptr [[VEC_GEP21]], align 8
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
-; CHECK-NEXT:    [[COL_LOAD22:%.*]] = load <1 x double>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP13]], ptr [[VEC_GEP21]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT:    [[COL_LOAD22:%.*]] = load <1 x double>, ptr [[TMP14]], align 8
 ; CHECK-NEXT:    [[VEC_GEP23:%.*]] = getelementptr double, ptr [[TMP3]], i64 5
 ; CHECK-NEXT:    [[COL_LOAD24:%.*]] = load <1 x double>, ptr [[VEC_GEP23]], align 8
 ; CHECK-NEXT:    [[COL_LOAD25:%.*]] = load <2 x double>, ptr [[TMP7]], align 8
 ; CHECK-NEXT:    [[VEC_GEP26:%.*]] = getelementptr double, ptr [[TMP7]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD27:%.*]] = load <2 x double>, ptr [[VEC_GEP26]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT29:%.*]] = shufflevector <2 x double> [[COL_LOAD25]], <2 x double> undef, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP13:%.*]] = fmul contract <1 x double> [[COL_LOAD22]], [[SPLAT_SPLATINSERT29]]
+; CHECK-NEXT:    [[TMP15:%.*]] = fmul contract <1 x double> [[COL_LOAD22]], [[SPLAT_SPLATINSERT29]]
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT32:%.*]] = shufflevector <2 x double> [[COL_LOAD25]], <2 x double> undef, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP14:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[SPLAT_SPLATINSERT32]], <1 x double> [[TMP13]])
+; CHECK-NEXT:    [[TMP16:%.*]] = fmul contract <1 x double> [[COL_LOAD24]], [[SPLAT_SPLATINSERT32]]
+; CHECK-NEXT:    [[TMP17:%.*]] = fadd contract <1 x double> [[TMP16]], [[TMP15]]
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT35:%.*]] = shufflevector <2 x double> [[COL_LOAD27]], <2 x double> undef, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP15:%.*]] = fmul contract <1 x double> [[COL_LOAD22]], [[SPLAT_SPLATINSERT35]]
+; CHECK-NEXT:    [[TMP18:%.*]] = fmul contract <1 x double> [[COL_LOAD22]], [[SPLAT_SPLATINSERT35]]
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT38:%.*]] = shufflevector <2 x double> [[COL_LOAD27]], <2 x double> undef, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP16:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[SPLAT_SPLATINSERT38]], <1 x double> [[TMP15]])
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr double, ptr [[C]], i64 2
-; CHECK-NEXT:    store <1 x double> [[TMP14]], ptr [[TMP17]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = fmul contract <1 x double> [[COL_LOAD24]], [[SPLAT_SPLATINSERT38]]
+; CHECK-NEXT:    [[TMP20:%.*]] = fadd contract <1 x double> [[TMP19]], [[TMP18]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr double, ptr [[C]], i64 2
+; CHECK-NEXT:    store <1 x double> [[TMP17]], ptr [[TMP21]], align 8
 ; CHECK-NEXT:    [[VEC_GEP40:%.*]] = getelementptr double, ptr [[C]], i64 5
-; CHECK-NEXT:    store <1 x double> [[TMP16]], ptr [[VEC_GEP40]], align 8
+; CHECK-NEXT:    store <1 x double> [[TMP20]], ptr [[VEC_GEP40]], align 8
 ; CHECK-NEXT:    [[COL_LOAD41:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
 ; CHECK-NEXT:    [[VEC_GEP42:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
 ; CHECK-NEXT:    [[COL_LOAD43:%.*]] = load <2 x double>, ptr [[VEC_GEP42]], align 8
-; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr double, ptr [[TMP7]], i64 4
-; CHECK-NEXT:    [[COL_LOAD44:%.*]] = load <2 x double>, ptr [[TMP18]], align 8
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr double, ptr [[TMP7]], i64 4
+; CHECK-NEXT:    [[COL_LOAD44:%.*]] = load <2 x double>, ptr [[TMP22]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT47:%.*]] = shufflevector <2 x double> [[COL_LOAD44]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP19:%.*]] = fmul contract <2 x double> [[COL_LOAD41]], [[SPLAT_SPLAT47]]
+; CHECK-NEXT:    [[TMP23:%.*]] = fmul contract <2 x double> [[COL_LOAD41]], [[SPLAT_SPLAT47]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT50:%.*]] = shufflevector <2 x double> [[COL_LOAD44]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP20:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD43]], <2 x double> [[SPLAT_SPLAT50]], <2 x double> [[TMP19]])
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr double, ptr [[C]], i64 6
-; CHECK-NEXT:    store <2 x double> [[TMP20]], ptr [[TMP21]], align 8
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
-; CHECK-NEXT:    [[COL_LOAD51:%.*]] = load <1 x double>, ptr [[TMP22]], align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = fmul contract <2 x double> [[COL_LOAD43]], [[SPLAT_SPLAT50]]
+; CHECK-NEXT:    [[TMP25:%.*]] = fadd contract <2 x double> [[TMP24]], [[TMP23]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr double, ptr [[C]], i64 6
+; CHECK-NEXT:    store <2 x double> [[TMP25]], ptr [[TMP26]], align 8
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT:    [[COL_LOAD51:%.*]] = load <1 x double>, ptr [[TMP27]], align 8
 ; CHECK-NEXT:    [[VEC_GEP52:%.*]] = getelementptr double, ptr [[TMP3]], i64 5
 ; CHECK-NEXT:    [[COL_LOAD53:%.*]] = load <1 x double>, ptr [[VEC_GEP52]], align 8
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr double, ptr [[TMP7]], i64 4
-; CHECK-NEXT:    [[COL_LOAD54:%.*]] = load <2 x double>, ptr [[TMP23]], align 8
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr double, ptr [[TMP7]], i64 4
+; CHECK-NEXT:    [[COL_LOAD54:%.*]] = load <2 x double>, ptr [[TMP28]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT56:%.*]] = shufflevector <2 x double> [[COL_LOAD54]], <2 x double> undef, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP24:%.*]] = fmul contract <1 x double> [[COL_LOAD51]], [[SPLAT_SPLATINSERT56]]
+; CHECK-NEXT:    [[TMP29:%.*]] = fmul contract <1 x double> [[COL_LOAD51]], [[SPLAT_SPLATINSERT56]]
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT59:%.*]] = shufflevector <2 x double> [[COL_LOAD54]], <2 x double> undef, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP25:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD53]], <1 x double> [[SPLAT_SPLATINSERT59]], <1 x double> [[TMP24]])
-; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr double, ptr [[C]], i64 8
-; CHECK-NEXT:    store <1 x double> [[TMP25]], ptr [[TMP26]], align 8
+; CHECK-NEXT:    [[TMP30:%.*]] = fmul contract <1 x double> [[COL_LOAD53]], [[SPLAT_SPLATINSERT59]]
+; CHECK-NEXT:    [[TMP31:%.*]] = fadd contract <1 x double> [[TMP30]], [[TMP29]]
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr double, ptr [[C]], i64 8
+; CHECK-NEXT:    store <1 x double> [[TMP31]], ptr [[TMP32]], align 8
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[TRUE:%.*]], label [[FALSE:%.*]]
 ; CHECK:       true:
-; CHECK-NEXT:    [[TMP27:%.*]] = fadd contract <3 x double> [[COL_LOAD133]], [[COL_LOAD133]]
-; CHECK-NEXT:    [[TMP28:%.*]] = fadd contract <3 x double> [[COL_LOAD135]], [[COL_LOAD135]]
-; CHECK-NEXT:    store <3 x double> [[TMP27]], ptr [[A]], align 8
+; CHECK-NEXT:    [[TMP33:%.*]] = fadd contract <3 x double> [[COL_LOAD133]], [[COL_LOAD133]]
+; CHECK-NEXT:    [[TMP34:%.*]] = fadd contract <3 x double> [[COL_LOAD135]], [[COL_LOAD135]]
+; CHECK-NEXT:    store <3 x double> [[TMP33]], ptr [[A]], align 8
 ; CHECK-NEXT:    [[VEC_GEP143:%.*]] = getelementptr double, ptr [[A]], i64 3
-; CHECK-NEXT:    store <3 x double> [[TMP28]], ptr [[VEC_GEP143]], align 8
+; CHECK-NEXT:    store <3 x double> [[TMP34]], ptr [[VEC_GEP143]], align 8
 ; CHECK-NEXT:    br label [[END:%.*]]
 ; CHECK:       false:
-; CHECK-NEXT:    [[TMP29:%.*]] = fadd contract <2 x double> [[COL_LOAD136]], [[COL_LOAD136]]
-; CHECK-NEXT:    [[TMP30:%.*]] = fadd contract <2 x double> [[COL_LOAD138]], [[COL_LOAD138]]
-; CHECK-NEXT:    [[TMP31:%.*]] = fadd contract <2 x double> [[COL_LOAD140]], [[COL_LOAD140]]
-; CHECK-NEXT:    store <2 x double> [[TMP29]], ptr [[B]], align 8
+; CHECK-NEXT:    [[TMP35:%.*]] = fadd contract <2 x double> [[COL_LOAD136]], [[COL_LOAD136]]
+; CHECK-NEXT:    [[TMP36:%.*]] = fadd contract <2 x double> [[COL_LOAD138]], [[COL_LOAD138]]
+; CHECK-NEXT:    [[TMP37:%.*]] = fadd contract <2 x double> [[COL_LOAD140]], [[COL_LOAD140]]
+; CHECK-NEXT:    store <2 x double> [[TMP35]], ptr [[B]], align 8
 ; CHECK-NEXT:    [[VEC_GEP141:%.*]] = getelementptr double, ptr [[B]], i64 2
-; CHECK-NEXT:    store <2 x double> [[TMP30]], ptr [[VEC_GEP141]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP36]], ptr [[VEC_GEP141]], align 8
 ; CHECK-NEXT:    [[VEC_GEP142:%.*]] = getelementptr double, ptr [[B]], i64 4
-; CHECK-NEXT:    store <2 x double> [[TMP31]], ptr [[VEC_GEP142]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP37]], ptr [[VEC_GEP142]], align 8
 ; CHECK-NEXT:    br label [[END]]
 ; CHECK:       end:
 ; CHECK-NEXT:    [[STORE_BEGIN64:%.*]] = ptrtoint ptr [[C]] to i64
 ; CHECK-NEXT:    [[STORE_END65:%.*]] = add nuw nsw i64 [[STORE_BEGIN64]], 72
 ; CHECK-NEXT:    [[LOAD_BEGIN66:%.*]] = ptrtoint ptr [[A]] to i64
-; CHECK-NEXT:    [[TMP32:%.*]] = icmp ugt i64 [[STORE_END65]], [[LOAD_BEGIN66]]
-; CHECK-NEXT:    br i1 [[TMP32]], label [[ALIAS_CONT61:%.*]], label [[NO_ALIAS63:%.*]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp ugt i64 [[STORE_END65]], [[LOAD_BEGIN66]]
+; CHECK-NEXT:    br i1 [[TMP38]], label [[ALIAS_CONT61:%.*]], label [[NO_ALIAS63:%.*]]
 ; CHECK:       alias_cont61:
 ; CHECK-NEXT:    [[LOAD_END67:%.*]] = add nuw nsw i64 [[LOAD_BEGIN66]], 48
-; CHECK-NEXT:    [[TMP33:%.*]] = icmp ugt i64 [[LOAD_END67]], [[STORE_BEGIN64]]
-; CHECK-NEXT:    br i1 [[TMP33]], label [[COPY62:%.*]], label [[NO_ALIAS63]]
+; CHECK-NEXT:    [[TMP39:%.*]] = icmp ugt i64 [[LOAD_END67]], [[STORE_BEGIN64]]
+; CHECK-NEXT:    br i1 [[TMP39]], label [[COPY62:%.*]], label [[NO_ALIAS63]]
 ; CHECK:       copy62:
-; CHECK-NEXT:    [[TMP34:%.*]] = alloca [6 x double], align 8
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(48) [[TMP34]], ptr noundef nonnull align 8 dereferenceable(48) [[A]], i64 48, i1 false)
+; CHECK-NEXT:    [[TMP40:%.*]] = alloca [6 x double], align 8
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(48) [[TMP40]], ptr noundef nonnull align 8 dereferenceable(48) [[A]], i64 48, i1 false)
 ; CHECK-NEXT:    br label [[NO_ALIAS63]]
 ; CHECK:       no_alias63:
-; CHECK-NEXT:    [[TMP35:%.*]] = phi ptr [ [[A]], [[END]] ], [ [[A]], [[ALIAS_CONT61]] ], [ [[TMP34]], [[COPY62]] ]
+; CHECK-NEXT:    [[TMP41:%.*]] = phi ptr [ [[A]], [[END]] ], [ [[A]], [[ALIAS_CONT61]] ], [ [[TMP40]], [[COPY62]] ]
 ; CHECK-NEXT:    [[STORE_BEGIN71:%.*]] = ptrtoint ptr [[C]] to i64
 ; CHECK-NEXT:    [[STORE_END72:%.*]] = add nuw nsw i64 [[STORE_BEGIN71]], 72
 ; CHECK-NEXT:    [[LOAD_BEGIN73:%.*]] = ptrtoint ptr [[B]] to i64
-; CHECK-NEXT:    [[TMP36:%.*]] = icmp ugt i64 [[STORE_END72]], [[LOAD_BEGIN73]]
-; CHECK-NEXT:    br i1 [[TMP36]], label [[ALIAS_CONT68:%.*]], label [[NO_ALIAS70:%.*]]
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ugt i64 [[STORE_END72]], [[LOAD_BEGIN73]]
+; CHECK-NEXT:    br i1 [[TMP42]], label [[ALIAS_CONT68:%.*]], label [[NO_ALIAS70:%.*]]
 ; CHECK:       alias_cont68:
 ; CHECK-NEXT:    [[LOAD_END74:%.*]] = add nuw nsw i64 [[LOAD_BEGIN73]], 48
-; CHECK-NEXT:    [[TMP37:%.*]] = icmp ugt i64 [[LOAD_END74]], [[STORE_BEGIN71]]
-; CHECK-NEXT:    br i1 [[TMP37]], label [[COPY69:%.*]], label [[NO_ALIAS70]]
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ugt i64 [[LOAD_END74]], [[STORE_BEGIN71]]
+; CHECK-NEXT:    br i1 [[TMP43]], label [[COPY69:%.*]], label [[NO_ALIAS70]]
 ; CHECK:       copy69:
-; CHECK-NEXT:    [[TMP38:%.*]] = alloca [6 x double], align 8
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(48) [[TMP38]], ptr noundef nonnull align 8 dereferenceable(48) [[B]], i64 48, i1 false)
+; CHECK-NEXT:    [[TMP44:%.*]] = alloca [6 x double], align 8
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(48) [[TMP44]], ptr noundef nonnull align 8 dereferenceable(48) [[B]], i64 48, i1 false)
 ; CHECK-NEXT:    br label [[NO_ALIAS70]]
 ; CHECK:       no_alias70:
-; CHECK-NEXT:    [[TMP39:%.*]] = phi ptr [ [[B]], [[NO_ALIAS63]] ], [ [[B]], [[ALIAS_CONT68]] ], [ [[TMP38]], [[COPY69]] ]
-; CHECK-NEXT:    [[COL_LOAD75:%.*]] = load <2 x double>, ptr [[TMP35]], align 8
-; CHECK-NEXT:    [[VEC_GEP76:%.*]] = getelementptr double, ptr [[TMP35]], i64 3
+; CHECK-NEXT:    [[TMP45:%.*]] = phi ptr [ [[B]], [[NO_ALIAS63]] ], [ [[B]], [[ALIAS_CONT68]] ], [ [[TMP44]], [[COPY69]] ]
+; CHECK-NEXT:    [[COL_LOAD75:%.*]] = load <2 x double>, ptr [[TMP41]], align 8
+; CHECK-NEXT:    [[VEC_GEP76:%.*]] = getelementptr double, ptr [[TMP41]], i64 3
 ; CHECK-NEXT:    [[COL_LOAD77:%.*]] = load <2 x double>, ptr [[VEC_GEP76]], align 8
-; CHECK-NEXT:    [[COL_LOAD78:%.*]] = load <2 x double>, ptr [[TMP39]], align 8
-; CHECK-NEXT:    [[VEC_GEP79:%.*]] = getelementptr double, ptr [[TMP39]], i64 2
+; CHECK-NEXT:    [[COL_LOAD78:%.*]] = load <2 x double>, ptr [[TMP45]], align 8
+; CHECK-NEXT:    [[VEC_GEP79:%.*]] = getelementptr double, ptr [[TMP45]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD80:%.*]] = load <2 x double>, ptr [[VEC_GEP79]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT83:%.*]] = shufflevector <2 x double> [[COL_LOAD78]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP40:%.*]] = fmul contract <2 x double> [[COL_LOAD75]], [[SPLAT_SPLAT83]]
+; CHECK-NEXT:    [[TMP46:%.*]] = fmul contract <2 x double> [[COL_LOAD75]], [[SPLAT_SPLAT83]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT86:%.*]] = shufflevector <2 x double> [[COL_LOAD78]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP41:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD77]], <2 x double> [[SPLAT_SPLAT86]], <2 x double> [[TMP40]])
+; CHECK-NEXT:    [[TMP47:%.*]] = fmul contract <2 x double> [[COL_LOAD77]], [[SPLAT_SPLAT86]]
+; CHECK-NEXT:    [[TMP48:%.*]] = fadd contract <2 x double> [[TMP47]], [[TMP46]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT89:%.*]] = shufflevector <2 x double> [[COL_LOAD80]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP42:%.*]] = fmul contract <2 x double> [[COL_LOAD75]], [[SPLAT_SPLAT89]]
+; CHECK-NEXT:    [[TMP49:%.*]] = fmul contract <2 x double> [[COL_LOAD75]], [[SPLAT_SPLAT89]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT92:%.*]] = shufflevector <2 x double> [[COL_LOAD80]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP43:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD77]], <2 x double> [[SPLAT_SPLAT92]], <2 x double> [[TMP42]])
-; CHECK-NEXT:    store <2 x double> [[TMP41]], ptr [[C]], align 8
+; CHECK-NEXT:    [[TMP50:%.*]] = fmul contract <2 x double> [[COL_LOAD77]], [[SPLAT_SPLAT92]]
+; CHECK-NEXT:    [[TMP51:%.*]] = fadd contract <2 x double> [[TMP50]], [[TMP49]]
+; CHECK-NEXT:    store <2 x double> [[TMP48]], ptr [[C]], align 8
 ; CHECK-NEXT:    [[VEC_GEP93:%.*]] = getelementptr double, ptr [[C]], i64 3
-; CHECK-NEXT:    store <2 x double> [[TMP43]], ptr [[VEC_GEP93]], align 8
-; CHECK-NEXT:    [[TMP44:%.*]] = getelementptr double, ptr [[TMP35]], i64 2
-; CHECK-NEXT:    [[COL_LOAD94:%.*]] = load <1 x double>, ptr [[TMP44]], align 8
-; CHECK-NEXT:    [[VEC_GEP95:%.*]] = getelementptr double, ptr [[TMP35]], i64 5
+; CHECK-NEXT:    store <2 x double> [[TMP51]], ptr [[VEC_GEP93]], align 8
+; CHECK-NEXT:    [[TMP52:%.*]] = getelementptr double, ptr [[TMP41]], i64 2
+; CHECK-NEXT:    [[COL_LOAD94:%.*]] = load <1 x double>, ptr [[TMP52]], align 8
+; CHECK-NEXT:    [[VEC_GEP95:%.*]] = getelementptr double, ptr [[TMP41]], i64 5
 ; CHECK-NEXT:    [[COL_LOAD96:%.*]] = load <1 x double>, ptr [[VEC_GEP95]], align 8
-; CHECK-NEXT:    [[COL_LOAD97:%.*]] = load <2 x double>, ptr [[TMP39]], align 8
-; CHECK-NEXT:    [[VEC_GEP98:%.*]] = getelementptr double, ptr [[TMP39]], i64 2
+; CHECK-NEXT:    [[COL_LOAD97:%.*]] = load <2 x double>, ptr [[TMP45]], align 8
+; CHECK-NEXT:    [[VEC_GEP98:%.*]] = getelementptr double, ptr [[TMP45]], i64 2
 ; CHECK-NEXT:    [[COL_LOAD99:%.*]] = load <2 x double>, ptr [[VEC_GEP98]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT101:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> undef, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP45:%.*]] = fmul contract <1 x double> [[COL_LOAD94]], [[SPLAT_SPLATINSERT101]]
+; CHECK-NEXT:    [[TMP53:%.*]] = fmul contract <1 x double> [[COL_LOAD94]], [[SPLAT_SPLATINSERT101]]
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT104:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> undef, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP46:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD96]], <1 x double> [[SPLAT_SPLATINSERT104]], <1 x double> [[TMP45]])
+; CHECK-NEXT:    [[TMP54:%.*]] = fmul contract <1 x double> [[COL_LOAD96]], [[SPLAT_SPLATINSERT104]]
+; CHECK-NEXT:    [[TMP55:%.*]] = fadd contract <1 x double> [[TMP54]], [[TMP53]]
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT107:%.*]] = shufflevector <2 x double> [[COL_LOAD99]], <2 x double> undef, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP47:%.*]] = fmul contract <1 x double> [[COL_LOAD94]], [[SPLAT_SPLATINSERT107]]
+; CHECK-NEXT:    [[TMP56:%.*]] = fmul contract <1 x double> [[COL_LOAD94]], [[SPLAT_SPLATINSERT107]]
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT110:%.*]] = shufflevector <2 x double> [[COL_LOAD99]], <2 x double> undef, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP48:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD96]], <1 x double> [[SPLAT_SPLATINSERT110]], <1 x double> [[TMP47]])
-; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr double, ptr [[C]], i64 2
-; CHECK-NEXT:    store <1 x double> [[TMP46]], ptr [[TMP49]], align 8
+; CHECK-NEXT:    [[TMP57:%.*]] = fmul contract <1 x double> [[COL_LOAD96]], [[SPLAT_SPLATINSERT110]]
+; CHECK-NEXT:    [[TMP58:%.*]] = fadd contract <1 x double> [[TMP57]], [[TMP56]]
+; CHECK-NEXT:    [[TMP59:%.*]] = getelementptr double, ptr [[C]], i64 2
+; CHECK-NEXT:    store <1 x double> [[TMP55]], ptr [[TMP59]], align 8
 ; CHECK-NEXT:    [[VEC_GEP112:%.*]] = getelementptr double, ptr [[C]], i64 5
-; CHECK-NEXT:    store <1 x double> [[TMP48]], ptr [[VEC_GEP112]], align 8
-; CHECK-NEXT:    [[COL_LOAD113:%.*]] = load <2 x double>, ptr [[TMP35]], align 8
-; CHECK-NEXT:    [[VEC_GEP114:%.*]] = getelementptr double, ptr [[TMP35]], i64 3
+; CHECK-NEXT:    store <1 x double> [[TMP58]], ptr [[VEC_GEP112]], align 8
+; CHECK-NEXT:    [[COL_LOAD113:%.*]] = load <2 x double>, ptr [[TMP41]], align 8
+; CHECK-NEXT:    [[VEC_GEP114:%.*]] = getelementptr double, ptr [[TMP41]], i64 3
 ; CHECK-NEXT:    [[COL_LOAD115:%.*]] = load <2 x double>, ptr [[VEC_GEP114]], align 8
-; CHECK-NEXT:    [[TMP50:%.*]] = getelementptr double, ptr [[TMP39]], i64 4
-; CHECK-NEXT:    [[COL_LOAD116:%.*]] = load <2 x double>, ptr [[TMP50]], align 8
+; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr double, ptr [[TMP45]], i64 4
+; CHECK-NEXT:    [[COL_LOAD116:%.*]] = load <2 x double>, ptr [[TMP60]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT119:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP51:%.*]] = fmul contract <2 x double> [[COL_LOAD113]], [[SPLAT_SPLAT119]]
+; CHECK-NEXT:    [[TMP61:%.*]] = fmul contract <2 x double> [[COL_LOAD113]], [[SPLAT_SPLAT119]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT122:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP52:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD115]], <2 x double> [[SPLAT_SPLAT122]], <2 x double> [[TMP51]])
-; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr double, ptr [[C]], i64 6
-; CHECK-NEXT:    store <2 x double> [[TMP52]], ptr [[TMP53]], align 8
-; CHECK-NEXT:    [[TMP54:%.*]] = getelementptr double, ptr [[TMP35]], i64 2
-; CHECK-NEXT:    [[COL_LOAD123:%.*]] = load <1 x double>, ptr [[TMP54]], align 8
-; CHECK-NEXT:    [[VEC_GEP124:%.*]] = getelementptr double, ptr [[TMP35]], i64 5
+; CHECK-NEXT:    [[TMP62:%.*]] = fmul contract <2 x double> [[COL_LOAD115]], [[SPLAT_SPLAT122]]
+; CHECK-NEXT:    [[TMP63:%.*]] = fadd contract <2 x double> [[TMP62]], [[TMP61]]
+; CHECK-NEXT:    [[TMP64:%.*]] = getelementptr double, ptr [[C]], i64 6
+; CHECK-NEXT:    store <2 x double> [[TMP63]], ptr [[TMP64]], align 8
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr double, ptr [[TMP41]], i64 2
+; CHECK-NEXT:    [[COL_LOAD123:%.*]] = load <1 x double>, ptr [[TMP65]], align 8
+; CHECK-NEXT:    [[VEC_GEP124:%.*]] = getelementptr double, ptr [[TMP41]], i64 5
 ; CHECK-NEXT:    [[COL_LOAD125:%.*]] = load <1 x double>, ptr [[VEC_GEP124]], align 8
-; CHECK-NEXT:    [[TMP55:%.*]] = getelementptr double, ptr [[TMP39]], i64 4
-; CHECK-NEXT:    [[COL_LOAD126:%.*]] = load <2 x double>, ptr [[TMP55]], align 8
+; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr double, ptr [[TMP45]], i64 4
+; CHECK-NEXT:    [[COL_LOAD126:%.*]] = load <2 x double>, ptr [[TMP66]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT128:%.*]] = shufflevector <2 x double> [[COL_LOAD126]], <2 x double> undef, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP56:%.*]] = fmul contract <1 x double> [[COL_LOAD123]], [[SPLAT_SPLATINSERT128]]
+; CHECK-NEXT:    [[TMP67:%.*]] = fmul contract <1 x double> [[COL_LOAD123]], [[SPLAT_SPLATINSERT128]]
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT131:%.*]] = shufflevector <2 x double> [[COL_LOAD126]], <2 x double> undef, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP57:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD125]], <1 x double> [[SPLAT_SPLATINSERT131]], <1 x double> [[TMP56]])
-; CHECK-NEXT:    [[TMP58:%.*]] = getelementptr double, ptr [[C]], i64 8
-; CHECK-NEXT:    store <1 x double> [[TMP57]], ptr [[TMP58]], align 8
+; CHECK-NEXT:    [[TMP68:%.*]] = fmul contract <1 x double> [[COL_LOAD125]], [[SPLAT_SPLATINSERT131]]
+; CHECK-NEXT:    [[TMP69:%.*]] = fadd contract <1 x double> [[TMP68]], [[TMP67]]
+; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr double, ptr [[C]], i64 8
+; CHECK-NEXT:    store <1 x double> [[TMP69]], ptr [[TMP70]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %a = load <6 x double>, ptr %A, align 8
   %b = load <6 x double>, ptr %B, align 8
   %c = call <9 x double> @llvm.matrix.multiply(<6 x double> %a, <6 x double> %b, i32 3, i32 2, i32 3)
   store <9 x double> %c, ptr %C, align 8
 

llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused.ll

 ; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[TMP3]], i64 4
 ; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
 ; CHECK-NEXT:    [[COL_LOAD9:%.*]] = load <2 x double>, ptr [[TMP7]], align 8
 ; CHECK-NEXT:    [[VEC_GEP10:%.*]] = getelementptr double, ptr [[TMP7]], i64 4
 ; CHECK-NEXT:    [[COL_LOAD11:%.*]] = load <2 x double>, ptr [[VEC_GEP10]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD9]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT14:%.*]] = shufflevector <2 x double> [[COL_LOAD9]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT14]], <2 x double> [[TMP8]])
+; CHECK-NEXT:    [[TMP9:%.*]] = fmul contract <2 x double> [[COL_LOAD8]], [[SPLAT_SPLAT14]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fadd contract <2 x double> [[TMP9]], [[TMP8]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT17:%.*]] = shufflevector <2 x double> [[COL_LOAD11]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT17]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT17]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT20:%.*]] = shufflevector <2 x double> [[COL_LOAD11]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP11:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT20]], <2 x double> [[TMP10]])
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr double, ptr [[TMP3]], i64 8
-; CHECK-NEXT:    [[COL_LOAD21:%.*]] = load <2 x double>, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul contract <2 x double> [[COL_LOAD8]], [[SPLAT_SPLAT20]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fadd contract <2 x double> [[TMP12]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr double, ptr [[TMP3]], i64 8
+; CHECK-NEXT:    [[COL_LOAD21:%.*]] = load <2 x double>, ptr [[TMP14]], align 8
 ; CHECK-NEXT:    [[VEC_GEP22:%.*]] = getelementptr double, ptr [[TMP3]], i64 12
 ; CHECK-NEXT:    [[COL_LOAD23:%.*]] = load <2 x double>, ptr [[VEC_GEP22]], align 8
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr double, ptr [[TMP7]], i64 2
-; CHECK-NEXT:    [[COL_LOAD24:%.*]] = load <2 x double>, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr double, ptr [[TMP7]], i64 2
+; CHECK-NEXT:    [[COL_LOAD24:%.*]] = load <2 x double>, ptr [[TMP15]], align 8
 ; CHECK-NEXT:    [[VEC_GEP25:%.*]] = getelementptr double, ptr [[TMP7]], i64 6
 ; CHECK-NEXT:    [[COL_LOAD26:%.*]] = load <2 x double>, ptr [[VEC_GEP25]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT30:%.*]] = shufflevector <2 x double> [[COL_LOAD24]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP14:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD21]], <2 x double> [[SPLAT_SPLAT30]], <2 x double> [[TMP9]])
+; CHECK-NEXT:    [[TMP16:%.*]] = fmul contract <2 x double> [[COL_LOAD21]], [[SPLAT_SPLAT30]]
+; CHECK-NEXT:    [[TMP17:%.*]] = fadd contract <2 x double> [[TMP16]], [[TMP10]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT33:%.*]] = shufflevector <2 x double> [[COL_LOAD24]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP15:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD23]], <2 x double> [[SPLAT_SPLAT33]], <2 x double> [[TMP14]])
+; CHECK-NEXT:    [[TMP18:%.*]] = fmul contract <2 x double> [[COL_LOAD23]], [[SPLAT_SPLAT33]]
+; CHECK-NEXT:    [[TMP19:%.*]] = fadd contract <2 x double> [[TMP18]], [[TMP17]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT37:%.*]] = shufflevector <2 x double> [[COL_LOAD26]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP16:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD21]], <2 x double> [[SPLAT_SPLAT37]], <2 x double> [[TMP11]])
+; CHECK-NEXT:    [[TMP20:%.*]] = fmul contract <2 x double> [[COL_LOAD21]], [[SPLAT_SPLAT37]]
+; CHECK-NEXT:    [[TMP21:%.*]] = fadd contract <2 x double> [[TMP20]], [[TMP13]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT40:%.*]] = shufflevector <2 x double> [[COL_LOAD26]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP17:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD23]], <2 x double> [[SPLAT_SPLAT40]], <2 x double> [[TMP16]])
-; CHECK-NEXT:    store <2 x double> [[TMP15]], ptr [[C]], align 8
+; CHECK-NEXT:    [[TMP22:%.*]] = fmul contract <2 x double> [[COL_LOAD23]], [[SPLAT_SPLAT40]]
+; CHECK-NEXT:    [[TMP23:%.*]] = fadd contract <2 x double> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    store <2 x double> [[TMP19]], ptr [[C]], align 8
 ; CHECK-NEXT:    [[VEC_GEP41:%.*]] = getelementptr double, ptr [[C]], i64 4
-; CHECK-NEXT:    store <2 x double> [[TMP17]], ptr [[VEC_GEP41]], align 8
-; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
-; CHECK-NEXT:    [[COL_LOAD42:%.*]] = load <2 x double>, ptr [[TMP18]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP23]], ptr [[VEC_GEP41]], align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT:    [[COL_LOAD42:%.*]] = load <2 x double>, ptr [[TMP24]], align 8
 ; CHECK-NEXT:    [[VEC_GEP43:%.*]] = getelementptr double, ptr [[TMP3]], i64 6
 ; CHECK-NEXT:    [[COL_LOAD44:%.*]] = load <2 x double>, ptr [[VEC_GEP43]], align 8
 ; CHECK-NEXT:    [[COL_LOAD45:%.*]] = load <2 x double>, ptr [[TMP7]], align 8
 ; CHECK-NEXT:    [[VEC_GEP46:%.*]] = getelementptr double, ptr [[TMP7]], i64 4
 ; CHECK-NEXT:    [[COL_LOAD47:%.*]] = load <2 x double>, ptr [[VEC_GEP46]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT50:%.*]] = shufflevector <2 x double> [[COL_LOAD45]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP19:%.*]] = fmul contract <2 x double> [[COL_LOAD42]], [[SPLAT_SPLAT50]]
+; CHECK-NEXT:    [[TMP25:%.*]] = fmul contract <2 x double> [[COL_LOAD42]], [[SPLAT_SPLAT50]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT53:%.*]] = shufflevector <2 x double> [[COL_LOAD45]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP20:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD44]], <2 x double> [[SPLAT_SPLAT53]], <2 x double> [[TMP19]])
+; CHECK-NEXT:    [[TMP26:%.*]] = fmul contract <2 x double> [[COL_LOAD44]], [[SPLAT_SPLAT53]]
+; CHECK-NEXT:    [[TMP27:%.*]] = fadd contract <2 x double> [[TMP26]], [[TMP25]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT56:%.*]] = shufflevector <2 x double> [[COL_LOAD47]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP21:%.*]] = fmul contract <2 x double> [[COL_LOAD42]], [[SPLAT_SPLAT56]]
+; CHECK-NEXT:    [[TMP28:%.*]] = fmul contract <2 x double> [[COL_LOAD42]], [[SPLAT_SPLAT56]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT59:%.*]] = shufflevector <2 x double> [[COL_LOAD47]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP22:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD44]], <2 x double> [[SPLAT_SPLAT59]], <2 x double> [[TMP21]])
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr double, ptr [[TMP3]], i64 10
-; CHECK-NEXT:    [[COL_LOAD60:%.*]] = load <2 x double>, ptr [[TMP23]], align 8
+; CHECK-NEXT:    [[TMP29:%.*]] = fmul contract <2 x double> [[COL_LOAD44]], [[SPLAT_SPLAT59]]
+; CHECK-NEXT:    [[TMP30:%.*]] = fadd contract <2 x double> [[TMP29]], [[TMP28]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr double, ptr [[TMP3]], i64 10
+; CHECK-NEXT:    [[COL_LOAD60:%.*]] = load <2 x double>, ptr [[TMP31]], align 8
 ; CHECK-NEXT:    [[VEC_GEP61:%.*]] = getelementptr double, ptr [[TMP3]], i64 14
 ; CHECK-NEXT:    [[COL_LOAD62:%.*]] = load <2 x double>, ptr [[VEC_GEP61]], align 8
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr double, ptr [[TMP7]], i64 2
-; CHECK-NEXT:    [[COL_LOAD63:%.*]] = load <2 x double>, ptr [[TMP24]], align 8
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr double, ptr [[TMP7]], i64 2
+; CHECK-NEXT:    [[COL_LOAD63:%.*]] = load <2 x double>, ptr [[TMP32]], align 8
 ; CHECK-NEXT:    [[VEC_GEP64:%.*]] = getelementptr double, ptr [[TMP7]], i64 6
 ; CHECK-NEXT:    [[COL_LOAD65:%.*]] = load <2 x double>, ptr [[VEC_GEP64]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT69:%.*]] = shufflevector <2 x double> [[COL_LOAD63]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP25:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD60]], <2 x double> [[SPLAT_SPLAT69]], <2 x double> [[TMP20]])
+; CHECK-NEXT:    [[TMP33:%.*]] = fmul contract <2 x double> [[COL_LOAD60]], [[SPLAT_SPLAT69]]
+; CHECK-NEXT:    [[TMP34:%.*]] = fadd contract <2 x double> [[TMP33]], [[TMP27]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT72:%.*]] = shufflevector <2 x double> [[COL_LOAD63]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP26:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD62]], <2 x double> [[SPLAT_SPLAT72]], <2 x double> [[TMP25]])
+; CHECK-NEXT:    [[TMP35:%.*]] = fmul contract <2 x double> [[COL_LOAD62]], [[SPLAT_SPLAT72]]
+; CHECK-NEXT:    [[TMP36:%.*]] = fadd contract <2 x double> [[TMP35]], [[TMP34]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT76:%.*]] = shufflevector <2 x double> [[COL_LOAD65]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP27:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD60]], <2 x double> [[SPLAT_SPLAT76]], <2 x double> [[TMP22]])
+; CHECK-NEXT:    [[TMP37:%.*]] = fmul contract <2 x double> [[COL_LOAD60]], [[SPLAT_SPLAT76]]
+; CHECK-NEXT:    [[TMP38:%.*]] = fadd contract <2 x double> [[TMP37]], [[TMP30]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT79:%.*]] = shufflevector <2 x double> [[COL_LOAD65]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP28:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD62]], <2 x double> [[SPLAT_SPLAT79]], <2 x double> [[TMP27]])
-; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr double, ptr [[C]], i64 2
-; CHECK-NEXT:    store <2 x double> [[TMP26]], ptr [[TMP29]], align 8
+; CHECK-NEXT:    [[TMP39:%.*]] = fmul contract <2 x double> [[COL_LOAD62]], [[SPLAT_SPLAT79]]
+; CHECK-NEXT:    [[TMP40:%.*]] = fadd contract <2 x double> [[TMP39]], [[TMP38]]
+; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr double, ptr [[C]], i64 2
+; CHECK-NEXT:    store <2 x double> [[TMP36]], ptr [[TMP41]], align 8
 ; CHECK-NEXT:    [[VEC_GEP80:%.*]] = getelementptr double, ptr [[C]], i64 6
-; CHECK-NEXT:    store <2 x double> [[TMP28]], ptr [[VEC_GEP80]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP40]], ptr [[VEC_GEP80]], align 8
 ; CHECK-NEXT:    [[COL_LOAD81:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
 ; CHECK-NEXT:    [[VEC_GEP82:%.*]] = getelementptr double, ptr [[TMP3]], i64 4
 ; CHECK-NEXT:    [[COL_LOAD83:%.*]] = load <2 x double>, ptr [[VEC_GEP82]], align 8
-; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr double, ptr [[TMP7]], i64 8
-; CHECK-NEXT:    [[COL_LOAD84:%.*]] = load <2 x double>, ptr [[TMP30]], align 8
+; CHECK-NEXT:    [[TMP42:%.*]] = getelementptr double, ptr [[TMP7]], i64 8
+; CHECK-NEXT:    [[COL_LOAD84:%.*]] = load <2 x double>, ptr [[TMP42]], align 8
 ; CHECK-NEXT:    [[VEC_GEP85:%.*]] = getelementptr double, ptr [[TMP7]], i64 12
 ; CHECK-NEXT:    [[COL_LOAD86:%.*]] = load <2 x double>, ptr [[VEC_GEP85]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT89:%.*]] = shufflevector <2 x double> [[COL_LOAD84]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP31:%.*]] = fmul contract <2 x double> [[COL_LOAD81]], [[SPLAT_SPLAT89]]
+; CHECK-NEXT:    [[TMP43:%.*]] = fmul contract <2 x double> [[COL_LOAD81]], [[SPLAT_SPLAT89]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT92:%.*]] = shufflevector <2 x double> [[COL_LOAD84]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP32:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD83]], <2 x double> [[SPLAT_SPLAT92]], <2 x double> [[TMP31]])
+; CHECK-NEXT:    [[TMP44:%.*]] = fmul contract <2 x double> [[COL_LOAD83]], [[SPLAT_SPLAT92]]
+; CHECK-NEXT:    [[TMP45:%.*]] = fadd contract <2 x double> [[TMP44]], [[TMP43]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT95:%.*]] = shufflevector <2 x double> [[COL_LOAD86]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP33:%.*]] = fmul contract <2 x double> [[COL_LOAD81]], [[SPLAT_SPLAT95]]
+; CHECK-NEXT:    [[TMP46:%.*]] = fmul contract <2 x double> [[COL_LOAD81]], [[SPLAT_SPLAT95]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT98:%.*]] = shufflevector <2 x double> [[COL_LOAD86]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP34:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD83]], <2 x double> [[SPLAT_SPLAT98]], <2 x double> [[TMP33]])
-; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr double, ptr [[TMP3]], i64 8
-; CHECK-NEXT:    [[COL_LOAD99:%.*]] = load <2 x double>, ptr [[TMP35]], align 8
+; CHECK-NEXT:    [[TMP47:%.*]] = fmul contract <2 x double> [[COL_LOAD83]], [[SPLAT_SPLAT98]]
+; CHECK-NEXT:    [[TMP48:%.*]] = fadd contract <2 x double> [[TMP47]], [[TMP46]]
+; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr double, ptr [[TMP3]], i64 8
+; CHECK-NEXT:    [[COL_LOAD99:%.*]] = load <2 x double>, ptr [[TMP49]], align 8
 ; CHECK-NEXT:    [[VEC_GEP100:%.*]] = getelementptr double, ptr [[TMP3]], i64 12
 ; CHECK-NEXT:    [[COL_LOAD101:%.*]] = load <2 x double>, ptr [[VEC_GEP100]], align 8
-; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr double, ptr [[TMP7]], i64 10
-; CHECK-NEXT:    [[COL_LOAD102:%.*]] = load <2 x double>, ptr [[TMP36]], align 8
+; CHECK-NEXT:    [[TMP50:%.*]] = getelementptr double, ptr [[TMP7]], i64 10
+; CHECK-NEXT:    [[COL_LOAD102:%.*]] = load <2 x double>, ptr [[TMP50]], align 8
 ; CHECK-NEXT:    [[VEC_GEP103:%.*]] = getelementptr double, ptr [[TMP7]], i64 14
 ; CHECK-NEXT:    [[COL_LOAD104:%.*]] = load <2 x double>, ptr [[VEC_GEP103]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT108:%.*]] = shufflevector <2 x double> [[COL_LOAD102]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP37:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD99]], <2 x double> [[SPLAT_SPLAT108]], <2 x double> [[TMP32]])
+; CHECK-NEXT:    [[TMP51:%.*]] = fmul contract <2 x double> [[COL_LOAD99]], [[SPLAT_SPLAT108]]
+; CHECK-NEXT:    [[TMP52:%.*]] = fadd contract <2 x double> [[TMP51]], [[TMP45]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT111:%.*]] = shufflevector <2 x double> [[COL_LOAD102]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP38:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD101]], <2 x double> [[SPLAT_SPLAT111]], <2 x double> [[TMP37]])
+; CHECK-NEXT:    [[TMP53:%.*]] = fmul contract <2 x double> [[COL_LOAD101]], [[SPLAT_SPLAT111]]
+; CHECK-NEXT:    [[TMP54:%.*]] = fadd contract <2 x double> [[TMP53]], [[TMP52]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT115:%.*]] = shufflevector <2 x double> [[COL_LOAD104]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP39:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD99]], <2 x double> [[SPLAT_SPLAT115]], <2 x double> [[TMP34]])
+; CHECK-NEXT:    [[TMP55:%.*]] = fmul contract <2 x double> [[COL_LOAD99]], [[SPLAT_SPLAT115]]
+; CHECK-NEXT:    [[TMP56:%.*]] = fadd contract <2 x double> [[TMP55]], [[TMP48]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT118:%.*]] = shufflevector <2 x double> [[COL_LOAD104]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP40:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD101]], <2 x double> [[SPLAT_SPLAT118]], <2 x double> [[TMP39]])
-; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr double, ptr [[C]], i64 8
-; CHECK-NEXT:    store <2 x double> [[TMP38]], ptr [[TMP41]], align 8
+; CHECK-NEXT:    [[TMP57:%.*]] = fmul contract <2 x double> [[COL_LOAD101]], [[SPLAT_SPLAT118]]
+; CHECK-NEXT:    [[TMP58:%.*]] = fadd contract <2 x double> [[TMP57]], [[TMP56]]
+; CHECK-NEXT:    [[TMP59:%.*]] = getelementptr double, ptr [[C]], i64 8
+; CHECK-NEXT:    store <2 x double> [[TMP54]], ptr [[TMP59]], align 8
 ; CHECK-NEXT:    [[VEC_GEP119:%.*]] = getelementptr double, ptr [[C]], i64 12
-; CHECK-NEXT:    store <2 x double> [[TMP40]], ptr [[VEC_GEP119]], align 8
-; CHECK-NEXT:    [[TMP42:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
-; CHECK-NEXT:    [[COL_LOAD120:%.*]] = load <2 x double>, ptr [[TMP42]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP58]], ptr [[VEC_GEP119]], align 8
+; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT:    [[COL_LOAD120:%.*]] = load <2 x double>, ptr [[TMP60]], align 8
 ; CHECK-NEXT:    [[VEC_GEP121:%.*]] = getelementptr double, ptr [[TMP3]], i64 6
 ; CHECK-NEXT:    [[COL_LOAD122:%.*]] = load <2 x double>, ptr [[VEC_GEP121]], align 8
-; CHECK-NEXT:    [[TMP43:%.*]] = getelementptr double, ptr [[TMP7]], i64 8
-; CHECK-NEXT:    [[COL_LOAD123:%.*]] = load <2 x double>, ptr [[TMP43]], align 8
+; CHECK-NEXT:    [[TMP61:%.*]] = getelementptr double, ptr [[TMP7]], i64 8
+; CHECK-NEXT:    [[COL_LOAD123:%.*]] = load <2 x double>, ptr [[TMP61]], align 8
 ; CHECK-NEXT:    [[VEC_GEP124:%.*]] = getelementptr double, ptr [[TMP7]], i64 12
 ; CHECK-NEXT:    [[COL_LOAD125:%.*]] = load <2 x double>, ptr [[VEC_GEP124]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT128:%.*]] = shufflevector <2 x double> [[COL_LOAD123]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP44:%.*]] = fmul contract <2 x double> [[COL_LOAD120]], [[SPLAT_SPLAT128]]
+; CHECK-NEXT:    [[TMP62:%.*]] = fmul contract <2 x double> [[COL_LOAD120]], [[SPLAT_SPLAT128]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT131:%.*]] = shufflevector <2 x double> [[COL_LOAD123]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP45:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD122]], <2 x double> [[SPLAT_SPLAT131]], <2 x double> [[TMP44]])
+; CHECK-NEXT:    [[TMP63:%.*]] = fmul contract <2 x double> [[COL_LOAD122]], [[SPLAT_SPLAT131]]
+; CHECK-NEXT:    [[TMP64:%.*]] = fadd contract <2 x double> [[TMP63]], [[TMP62]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT134:%.*]] = shufflevector <2 x double> [[COL_LOAD125]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP46:%.*]] = fmul contract <2 x double> [[COL_LOAD120]], [[SPLAT_SPLAT134]]
+; CHECK-NEXT:    [[TMP65:%.*]] = fmul contract <2 x double> [[COL_LOAD120]], [[SPLAT_SPLAT134]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT137:%.*]] = shufflevector <2 x double> [[COL_LOAD125]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP47:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD122]], <2 x double> [[SPLAT_SPLAT137]], <2 x double> [[TMP46]])
-; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr double, ptr [[TMP3]], i64 10
-; CHECK-NEXT:    [[COL_LOAD138:%.*]] = load <2 x double>, ptr [[TMP48]], align 8
+; CHECK-NEXT:    [[TMP66:%.*]] = fmul contract <2 x double> [[COL_LOAD122]], [[SPLAT_SPLAT137]]
+; CHECK-NEXT:    [[TMP67:%.*]] = fadd contract <2 x double> [[TMP66]], [[TMP65]]
+; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr double, ptr [[TMP3]], i64 10
+; CHECK-NEXT:    [[COL_LOAD138:%.*]] = load <2 x double>, ptr [[TMP68]], align 8
 ; CHECK-NEXT:    [[VEC_GEP139:%.*]] = getelementptr double, ptr [[TMP3]], i64 14
 ; CHECK-NEXT:    [[COL_LOAD140:%.*]] = load <2 x double>, ptr [[VEC_GEP139]], align 8
-; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr double, ptr [[TMP7]], i64 10
-; CHECK-NEXT:    [[COL_LOAD141:%.*]] = load <2 x double>, ptr [[TMP49]], align 8
+; CHECK-NEXT:    [[TMP69:%.*]] = getelementptr double, ptr [[TMP7]], i64 10
+; CHECK-NEXT:    [[COL_LOAD141:%.*]] = load <2 x double>, ptr [[TMP69]], align 8
 ; CHECK-NEXT:    [[VEC_GEP142:%.*]] = getelementptr double, ptr [[TMP7]], i64 14
 ; CHECK-NEXT:    [[COL_LOAD143:%.*]] = load <2 x double>, ptr [[VEC_GEP142]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT147:%.*]] = shufflevector <2 x double> [[COL_LOAD141]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP50:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD138]], <2 x double> [[SPLAT_SPLAT147]], <2 x double> [[TMP45]])
+; CHECK-NEXT:    [[TMP70:%.*]] = fmul contract <2 x double> [[COL_LOAD138]], [[SPLAT_SPLAT147]]
+; CHECK-NEXT:    [[TMP71:%.*]] = fadd contract <2 x double> [[TMP70]], [[TMP64]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT150:%.*]] = shufflevector <2 x double> [[COL_LOAD141]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP51:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD140]], <2 x double> [[SPLAT_SPLAT150]], <2 x double> [[TMP50]])
+; CHECK-NEXT:    [[TMP72:%.*]] = fmul contract <2 x double> [[COL_LOAD140]], [[SPLAT_SPLAT150]]
+; CHECK-NEXT:    [[TMP73:%.*]] = fadd contract <2 x double> [[TMP72]], [[TMP71]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT154:%.*]] = shufflevector <2 x double> [[COL_LOAD143]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP52:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD138]], <2 x double> [[SPLAT_SPLAT154]], <2 x double> [[TMP47]])
+; CHECK-NEXT:    [[TMP74:%.*]] = fmul contract <2 x double> [[COL_LOAD138]], [[SPLAT_SPLAT154]]
+; CHECK-NEXT:    [[TMP75:%.*]] = fadd contract <2 x double> [[TMP74]], [[TMP67]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT157:%.*]] = shufflevector <2 x double> [[COL_LOAD143]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP53:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD140]], <2 x double> [[SPLAT_SPLAT157]], <2 x double> [[TMP52]])
-; CHECK-NEXT:    [[TMP54:%.*]] = getelementptr double, ptr [[C]], i64 10
-; CHECK-NEXT:    store <2 x double> [[TMP51]], ptr [[TMP54]], align 8
+; CHECK-NEXT:    [[TMP76:%.*]] = fmul contract <2 x double> [[COL_LOAD140]], [[SPLAT_SPLAT157]]
+; CHECK-NEXT:    [[TMP77:%.*]] = fadd contract <2 x double> [[TMP76]], [[TMP75]]
+; CHECK-NEXT:    [[TMP78:%.*]] = getelementptr double, ptr [[C]], i64 10
+; CHECK-NEXT:    store <2 x double> [[TMP73]], ptr [[TMP78]], align 8
 ; CHECK-NEXT:    [[VEC_GEP158:%.*]] = getelementptr double, ptr [[C]], i64 14
-; CHECK-NEXT:    store <2 x double> [[TMP53]], ptr [[VEC_GEP158]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP77]], ptr [[VEC_GEP158]], align 8
 ; CHECK-NEXT:    ret void
 ;
 
 
 ;; np.dot(a[0:2, 0:2], b[0:2, 0:2])
 
 
 ;; + np.dot(a[0:2, 2:4], b[2:4, 0:2])

 ; CHECK-LABEL: @multiply_reuse_load(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
 ; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 4
 ; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul contract <2 x double> [[COL_LOAD1]], [[SPLAT_SPLAT7]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd contract <2 x double> [[TMP1]], [[TMP0]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]])
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr double, ptr [[A]], i64 8
-; CHECK-NEXT:    [[COL_LOAD14:%.*]] = load <2 x double>, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul contract <2 x double> [[COL_LOAD1]], [[SPLAT_SPLAT13]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd contract <2 x double> [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr double, ptr [[A]], i64 8
+; CHECK-NEXT:    [[COL_LOAD14:%.*]] = load <2 x double>, ptr [[TMP6]], align 8
 ; CHECK-NEXT:    [[VEC_GEP15:%.*]] = getelementptr double, ptr [[A]], i64 12
 ; CHECK-NEXT:    [[COL_LOAD16:%.*]] = load <2 x double>, ptr [[VEC_GEP15]], align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, ptr [[A]], i64 2
-; CHECK-NEXT:    [[COL_LOAD17:%.*]] = load <2 x double>, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT:    [[COL_LOAD17:%.*]] = load <2 x double>, ptr [[TMP7]], align 8
 ; CHECK-NEXT:    [[VEC_GEP18:%.*]] = getelementptr double, ptr [[A]], i64 6
 ; CHECK-NEXT:    [[COL_LOAD19:%.*]] = load <2 x double>, ptr [[VEC_GEP18]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT23:%.*]] = shufflevector <2 x double> [[COL_LOAD17]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD14]], <2 x double> [[SPLAT_SPLAT23]], <2 x double> [[TMP1]])
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul contract <2 x double> [[COL_LOAD14]], [[SPLAT_SPLAT23]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fadd contract <2 x double> [[TMP8]], [[TMP2]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT26:%.*]] = shufflevector <2 x double> [[COL_LOAD17]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD16]], <2 x double> [[SPLAT_SPLAT26]], <2 x double> [[TMP6]])
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul contract <2 x double> [[COL_LOAD16]], [[SPLAT_SPLAT26]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd contract <2 x double> [[TMP10]], [[TMP9]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT30:%.*]] = shufflevector <2 x double> [[COL_LOAD19]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP8:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD14]], <2 x double> [[SPLAT_SPLAT30]], <2 x double> [[TMP3]])
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul contract <2 x double> [[COL_LOAD14]], [[SPLAT_SPLAT30]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fadd contract <2 x double> [[TMP12]], [[TMP5]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT33:%.*]] = shufflevector <2 x double> [[COL_LOAD19]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD16]], <2 x double> [[SPLAT_SPLAT33]], <2 x double> [[TMP8]])
-; CHECK-NEXT:    store <2 x double> [[TMP7]], ptr [[C:%.*]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = fmul contract <2 x double> [[COL_LOAD16]], [[SPLAT_SPLAT33]]
+; CHECK-NEXT:    [[TMP15:%.*]] = fadd contract <2 x double> [[TMP14]], [[TMP13]]
+; CHECK-NEXT:    store <2 x double> [[TMP11]], ptr [[C:%.*]], align 8
 ; CHECK-NEXT:    [[VEC_GEP34:%.*]] = getelementptr double, ptr [[C]], i64 4
-; CHECK-NEXT:    store <2 x double> [[TMP9]], ptr [[VEC_GEP34]], align 8
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr double, ptr [[A]], i64 2
-; CHECK-NEXT:    [[COL_LOAD35:%.*]] = load <2 x double>, ptr [[TMP10]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP15]], ptr [[VEC_GEP34]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT:    [[COL_LOAD35:%.*]] = load <2 x double>, ptr [[TMP16]], align 8
 ; CHECK-NEXT:    [[VEC_GEP36:%.*]] = getelementptr double, ptr [[A]], i64 6
 ; CHECK-NEXT:    [[COL_LOAD37:%.*]] = load <2 x double>, ptr [[VEC_GEP36]], align 8
 ; CHECK-NEXT:    [[COL_LOAD38:%.*]] = load <2 x double>, ptr [[A]], align 8
 ; CHECK-NEXT:    [[VEC_GEP39:%.*]] = getelementptr double, ptr [[A]], i64 4
 ; CHECK-NEXT:    [[COL_LOAD40:%.*]] = load <2 x double>, ptr [[VEC_GEP39]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT43:%.*]] = shufflevector <2 x double> [[COL_LOAD38]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP11:%.*]] = fmul contract <2 x double> [[COL_LOAD35]], [[SPLAT_SPLAT43]]
+; CHECK-NEXT:    [[TMP17:%.*]] = fmul contract <2 x double> [[COL_LOAD35]], [[SPLAT_SPLAT43]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT46:%.*]] = shufflevector <2 x double> [[COL_LOAD38]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP12:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD37]], <2 x double> [[SPLAT_SPLAT46]], <2 x double> [[TMP11]])
+; CHECK-NEXT:    [[TMP18:%.*]] = fmul contract <2 x double> [[COL_LOAD37]], [[SPLAT_SPLAT46]]
+; CHECK-NEXT:    [[TMP19:%.*]] = fadd contract <2 x double> [[TMP18]], [[TMP17]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT49:%.*]] = shufflevector <2 x double> [[COL_LOAD40]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP13:%.*]] = fmul contract <2 x double> [[COL_LOAD35]], [[SPLAT_SPLAT49]]
+; CHECK-NEXT:    [[TMP20:%.*]] = fmul contract <2 x double> [[COL_LOAD35]], [[SPLAT_SPLAT49]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT52:%.*]] = shufflevector <2 x double> [[COL_LOAD40]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP14:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD37]], <2 x double> [[SPLAT_SPLAT52]], <2 x double> [[TMP13]])
-; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr double, ptr [[A]], i64 10
-; CHECK-NEXT:    [[COL_LOAD53:%.*]] = load <2 x double>, ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = fmul contract <2 x double> [[COL_LOAD37]], [[SPLAT_SPLAT52]]
+; CHECK-NEXT:    [[TMP22:%.*]] = fadd contract <2 x double> [[TMP21]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr double, ptr [[A]], i64 10
+; CHECK-NEXT:    [[COL_LOAD53:%.*]] = load <2 x double>, ptr [[TMP23]], align 8
 ; CHECK-NEXT:    [[VEC_GEP54:%.*]] = getelementptr double, ptr [[A]], i64 14
 ; CHECK-NEXT:    [[COL_LOAD55:%.*]] = load <2 x double>, ptr [[VEC_GEP54]], align 8
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr double, ptr [[A]], i64 2
-; CHECK-NEXT:    [[COL_LOAD56:%.*]] = load <2 x double>, ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT:    [[COL_LOAD56:%.*]] = load <2 x double>, ptr [[TMP24]], align 8
 ; CHECK-NEXT:    [[VEC_GEP57:%.*]] = getelementptr double, ptr [[A]], i64 6
 ; CHECK-NEXT:    [[COL_LOAD58:%.*]] = load <2 x double>, ptr [[VEC_GEP57]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT62:%.*]] = shufflevector <2 x double> [[COL_LOAD56]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP17:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD53]], <2 x double> [[SPLAT_SPLAT62]], <2 x double> [[TMP12]])
+; CHECK-NEXT:    [[TMP25:%.*]] = fmul contract <2 x double> [[COL_LOAD53]], [[SPLAT_SPLAT62]]
+; CHECK-NEXT:    [[TMP26:%.*]] = fadd contract <2 x double> [[TMP25]], [[TMP19]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT65:%.*]] = shufflevector <2 x double> [[COL_LOAD56]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP18:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD55]], <2 x double> [[SPLAT_SPLAT65]], <2 x double> [[TMP17]])
+; CHECK-NEXT:    [[TMP27:%.*]] = fmul contract <2 x double> [[COL_LOAD55]], [[SPLAT_SPLAT65]]
+; CHECK-NEXT:    [[TMP28:%.*]] = fadd contract <2 x double> [[TMP27]], [[TMP26]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT69:%.*]] = shufflevector <2 x double> [[COL_LOAD58]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP19:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD53]], <2 x double> [[SPLAT_SPLAT69]], <2 x double> [[TMP14]])
+; CHECK-NEXT:    [[TMP29:%.*]] = fmul contract <2 x double> [[COL_LOAD53]], [[SPLAT_SPLAT69]]
+; CHECK-NEXT:    [[TMP30:%.*]] = fadd contract <2 x double> [[TMP29]], [[TMP22]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT72:%.*]] = shufflevector <2 x double> [[COL_LOAD58]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP20:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD55]], <2 x double> [[SPLAT_SPLAT72]], <2 x double> [[TMP19]])
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr double, ptr [[C]], i64 2
-; CHECK-NEXT:    store <2 x double> [[TMP18]], ptr [[TMP21]], align 8
+; CHECK-NEXT:    [[TMP31:%.*]] = fmul contract <2 x double> [[COL_LOAD55]], [[SPLAT_SPLAT72]]
+; CHECK-NEXT:    [[TMP32:%.*]] = fadd contract <2 x double> [[TMP31]], [[TMP30]]
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr double, ptr [[C]], i64 2
+; CHECK-NEXT:    store <2 x double> [[TMP28]], ptr [[TMP33]], align 8
 ; CHECK-NEXT:    [[VEC_GEP73:%.*]] = getelementptr double, ptr [[C]], i64 6
-; CHECK-NEXT:    store <2 x double> [[TMP20]], ptr [[VEC_GEP73]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP32]], ptr [[VEC_GEP73]], align 8
 ; CHECK-NEXT:    [[COL_LOAD74:%.*]] = load <2 x double>, ptr [[A]], align 8
 ; CHECK-NEXT:    [[VEC_GEP75:%.*]] = getelementptr double, ptr [[A]], i64 4
 ; CHECK-NEXT:    [[COL_LOAD76:%.*]] = load <2 x double>, ptr [[VEC_GEP75]], align 8
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr double, ptr [[A]], i64 8
-; CHECK-NEXT:    [[COL_LOAD77:%.*]] = load <2 x double>, ptr [[TMP22]], align 8
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr double, ptr [[A]], i64 8
+; CHECK-NEXT:    [[COL_LOAD77:%.*]] = load <2 x double>, ptr [[TMP34]], align 8
 ; CHECK-NEXT:    [[VEC_GEP78:%.*]] = getelementptr double, ptr [[A]], i64 12
 ; CHECK-NEXT:    [[COL_LOAD79:%.*]] = load <2 x double>, ptr [[VEC_GEP78]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT82:%.*]] = shufflevector <2 x double> [[COL_LOAD77]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP23:%.*]] = fmul contract <2 x double> [[COL_LOAD74]], [[SPLAT_SPLAT82]]
+; CHECK-NEXT:    [[TMP35:%.*]] = fmul contract <2 x double> [[COL_LOAD74]], [[SPLAT_SPLAT82]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT85:%.*]] = shufflevector <2 x double> [[COL_LOAD77]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP24:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD76]], <2 x double> [[SPLAT_SPLAT85]], <2 x double> [[TMP23]])
+; CHECK-NEXT:    [[TMP36:%.*]] = fmul contract <2 x double> [[COL_LOAD76]], [[SPLAT_SPLAT85]]
+; CHECK-NEXT:    [[TMP37:%.*]] = fadd contract <2 x double> [[TMP36]], [[TMP35]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT88:%.*]] = shufflevector <2 x double> [[COL_LOAD79]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP25:%.*]] = fmul contract <2 x double> [[COL_LOAD74]], [[SPLAT_SPLAT88]]
+; CHECK-NEXT:    [[TMP38:%.*]] = fmul contract <2 x double> [[COL_LOAD74]], [[SPLAT_SPLAT88]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT91:%.*]] = shufflevector <2 x double> [[COL_LOAD79]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP26:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD76]], <2 x double> [[SPLAT_SPLAT91]], <2 x double> [[TMP25]])
-; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr double, ptr [[A]], i64 8
-; CHECK-NEXT:    [[COL_LOAD92:%.*]] = load <2 x double>, ptr [[TMP27]], align 8
+; CHECK-NEXT:    [[TMP39:%.*]] = fmul contract <2 x double> [[COL_LOAD76]], [[SPLAT_SPLAT91]]
+; CHECK-NEXT:    [[TMP40:%.*]] = fadd contract <2 x double> [[TMP39]], [[TMP38]]
+; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr double, ptr [[A]], i64 8
+; CHECK-NEXT:    [[COL_LOAD92:%.*]] = load <2 x double>, ptr [[TMP41]], align 8
 ; CHECK-NEXT:    [[VEC_GEP93:%.*]] = getelementptr double, ptr [[A]], i64 12
 ; CHECK-NEXT:    [[COL_LOAD94:%.*]] = load <2 x double>, ptr [[VEC_GEP93]], align 8
-; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr double, ptr [[A]], i64 10
-; CHECK-NEXT:    [[COL_LOAD95:%.*]] = load <2 x double>, ptr [[TMP28]], align 8
+; CHECK-NEXT:    [[TMP42:%.*]] = getelementptr double, ptr [[A]], i64 10
+; CHECK-NEXT:    [[COL_LOAD95:%.*]] = load <2 x double>, ptr [[TMP42]], align 8
 ; CHECK-NEXT:    [[VEC_GEP96:%.*]] = getelementptr double, ptr [[A]], i64 14
 ; CHECK-NEXT:    [[COL_LOAD97:%.*]] = load <2 x double>, ptr [[VEC_GEP96]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT101:%.*]] = shufflevector <2 x double> [[COL_LOAD95]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP29:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD92]], <2 x double> [[SPLAT_SPLAT101]], <2 x double> [[TMP24]])
+; CHECK-NEXT:    [[TMP43:%.*]] = fmul contract <2 x double> [[COL_LOAD92]], [[SPLAT_SPLAT101]]
+; CHECK-NEXT:    [[TMP44:%.*]] = fadd contract <2 x double> [[TMP43]], [[TMP37]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT104:%.*]] = shufflevector <2 x double> [[COL_LOAD95]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP30:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD94]], <2 x double> [[SPLAT_SPLAT104]], <2 x double> [[TMP29]])
+; CHECK-NEXT:    [[TMP45:%.*]] = fmul contract <2 x double> [[COL_LOAD94]], [[SPLAT_SPLAT104]]
+; CHECK-NEXT:    [[TMP46:%.*]] = fadd contract <2 x double> [[TMP45]], [[TMP44]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT108:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP31:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD92]], <2 x double> [[SPLAT_SPLAT108]], <2 x double> [[TMP26]])
+; CHECK-NEXT:    [[TMP47:%.*]] = fmul contract <2 x double> [[COL_LOAD92]], [[SPLAT_SPLAT108]]
+; CHECK-NEXT:    [[TMP48:%.*]] = fadd contract <2 x double> [[TMP47]], [[TMP40]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT111:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP32:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD94]], <2 x double> [[SPLAT_SPLAT111]], <2 x double> [[TMP31]])
-; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr double, ptr [[C]], i64 8
-; CHECK-NEXT:    store <2 x double> [[TMP30]], ptr [[TMP33]], align 8
+; CHECK-NEXT:    [[TMP49:%.*]] = fmul contract <2 x double> [[COL_LOAD94]], [[SPLAT_SPLAT111]]
+; CHECK-NEXT:    [[TMP50:%.*]] = fadd contract <2 x double> [[TMP49]], [[TMP48]]
+; CHECK-NEXT:    [[TMP51:%.*]] = getelementptr double, ptr [[C]], i64 8
+; CHECK-NEXT:    store <2 x double> [[TMP46]], ptr [[TMP51]], align 8
 ; CHECK-NEXT:    [[VEC_GEP112:%.*]] = getelementptr double, ptr [[C]], i64 12
-; CHECK-NEXT:    store <2 x double> [[TMP32]], ptr [[VEC_GEP112]], align 8
-; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr double, ptr [[A]], i64 2
-; CHECK-NEXT:    [[COL_LOAD113:%.*]] = load <2 x double>, ptr [[TMP34]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP50]], ptr [[VEC_GEP112]], align 8
+; CHECK-NEXT:    [[TMP52:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT:    [[COL_LOAD113:%.*]] = load <2 x double>, ptr [[TMP52]], align 8
 ; CHECK-NEXT:    [[VEC_GEP114:%.*]] = getelementptr double, ptr [[A]], i64 6
 ; CHECK-NEXT:    [[COL_LOAD115:%.*]] = load <2 x double>, ptr [[VEC_GEP114]], align 8
-; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr double, ptr [[A]], i64 8
-; CHECK-NEXT:    [[COL_LOAD116:%.*]] = load <2 x double>, ptr [[TMP35]], align 8
+; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr double, ptr [[A]], i64 8
+; CHECK-NEXT:    [[COL_LOAD116:%.*]] = load <2 x double>, ptr [[TMP53]], align 8
 ; CHECK-NEXT:    [[VEC_GEP117:%.*]] = getelementptr double, ptr [[A]], i64 12
 ; CHECK-NEXT:    [[COL_LOAD118:%.*]] = load <2 x double>, ptr [[VEC_GEP117]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT121:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP36:%.*]] = fmul contract <2 x double> [[COL_LOAD113]], [[SPLAT_SPLAT121]]
+; CHECK-NEXT:    [[TMP54:%.*]] = fmul contract <2 x double> [[COL_LOAD113]], [[SPLAT_SPLAT121]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT124:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP37:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD115]], <2 x double> [[SPLAT_SPLAT124]], <2 x double> [[TMP36]])
+; CHECK-NEXT:    [[TMP55:%.*]] = fmul contract <2 x double> [[COL_LOAD115]], [[SPLAT_SPLAT124]]
+; CHECK-NEXT:    [[TMP56:%.*]] = fadd contract <2 x double> [[TMP55]], [[TMP54]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT127:%.*]] = shufflevector <2 x double> [[COL_LOAD118]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP38:%.*]] = fmul contract <2 x double> [[COL_LOAD113]], [[SPLAT_SPLAT127]]
+; CHECK-NEXT:    [[TMP57:%.*]] = fmul contract <2 x double> [[COL_LOAD113]], [[SPLAT_SPLAT127]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT130:%.*]] = shufflevector <2 x double> [[COL_LOAD118]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP39:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD115]], <2 x double> [[SPLAT_SPLAT130]], <2 x double> [[TMP38]])
-; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr double, ptr [[A]], i64 10
-; CHECK-NEXT:    [[COL_LOAD131:%.*]] = load <2 x double>, ptr [[TMP40]], align 8
+; CHECK-NEXT:    [[TMP58:%.*]] = fmul contract <2 x double> [[COL_LOAD115]], [[SPLAT_SPLAT130]]
+; CHECK-NEXT:    [[TMP59:%.*]] = fadd contract <2 x double> [[TMP58]], [[TMP57]]
+; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr double, ptr [[A]], i64 10
+; CHECK-NEXT:    [[COL_LOAD131:%.*]] = load <2 x double>, ptr [[TMP60]], align 8
 ; CHECK-NEXT:    [[VEC_GEP132:%.*]] = getelementptr double, ptr [[A]], i64 14
 ; CHECK-NEXT:    [[COL_LOAD133:%.*]] = load <2 x double>, ptr [[VEC_GEP132]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT140:%.*]] = shufflevector <2 x double> [[COL_LOAD131]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP41:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD131]], <2 x double> [[SPLAT_SPLAT140]], <2 x double> [[TMP37]])
+; CHECK-NEXT:    [[TMP61:%.*]] = fmul contract <2 x double> [[COL_LOAD131]], [[SPLAT_SPLAT140]]
+; CHECK-NEXT:    [[TMP62:%.*]] = fadd contract <2 x double> [[TMP61]], [[TMP56]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT143:%.*]] = shufflevector <2 x double> [[COL_LOAD131]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP42:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD133]], <2 x double> [[SPLAT_SPLAT143]], <2 x double> [[TMP41]])
+; CHECK-NEXT:    [[TMP63:%.*]] = fmul contract <2 x double> [[COL_LOAD133]], [[SPLAT_SPLAT143]]
+; CHECK-NEXT:    [[TMP64:%.*]] = fadd contract <2 x double> [[TMP63]], [[TMP62]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT147:%.*]] = shufflevector <2 x double> [[COL_LOAD133]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP43:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD131]], <2 x double> [[SPLAT_SPLAT147]], <2 x double> [[TMP39]])
+; CHECK-NEXT:    [[TMP65:%.*]] = fmul contract <2 x double> [[COL_LOAD131]], [[SPLAT_SPLAT147]]
+; CHECK-NEXT:    [[TMP66:%.*]] = fadd contract <2 x double> [[TMP65]], [[TMP59]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT150:%.*]] = shufflevector <2 x double> [[COL_LOAD133]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP44:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD133]], <2 x double> [[SPLAT_SPLAT150]], <2 x double> [[TMP43]])
-; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr double, ptr [[C]], i64 10
-; CHECK-NEXT:    store <2 x double> [[TMP42]], ptr [[TMP45]], align 8
+; CHECK-NEXT:    [[TMP67:%.*]] = fmul contract <2 x double> [[COL_LOAD133]], [[SPLAT_SPLAT150]]
+; CHECK-NEXT:    [[TMP68:%.*]] = fadd contract <2 x double> [[TMP67]], [[TMP66]]
+; CHECK-NEXT:    [[TMP69:%.*]] = getelementptr double, ptr [[C]], i64 10
+; CHECK-NEXT:    store <2 x double> [[TMP64]], ptr [[TMP69]], align 8
 ; CHECK-NEXT:    [[VEC_GEP151:%.*]] = getelementptr double, ptr [[C]], i64 14
-; CHECK-NEXT:    store <2 x double> [[TMP44]], ptr [[VEC_GEP151]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP68]], ptr [[VEC_GEP151]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %a = load <16 x double>, ptr %A, align 8
   %c = call <16 x double> @llvm.matrix.multiply(<16 x double> %a, <16 x double> %a, i32 4, i32 4, i32 4)
   store <16 x double> %c, ptr %C, align 8
   ret void
 }
 
 declare <16 x double> @llvm.matrix.multiply(<16 x double>, <16 x double>, i32, i32, i32)

llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll

 ; CHECK-NEXT:    [[VEC_GEP4:%.*]] = getelementptr double, ptr [[A]], i64 6
 ; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <2 x double>, ptr [[VEC_GEP4]], align 8
 ; CHECK-NEXT:    [[COL_LOAD6:%.*]] = load <4 x double>, ptr [[B:%.*]], align 8
 ; CHECK-NEXT:    [[VEC_GEP7:%.*]] = getelementptr double, ptr [[B]], i64 4
 ; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <4 x double>, ptr [[VEC_GEP7]], align 8
 ; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> undef, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT11]], <2 x double> [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul contract <2 x double> [[COL_LOAD1]], [[SPLAT_SPLAT11]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd contract <2 x double> [[TMP1]], [[TMP0]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT14:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> undef, <2 x i32> <i32 2, i32 2>
-; CHECK-NEXT:    [[TMP2:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD3]], <2 x double> [[SPLAT_SPLAT14]], <2 x double> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul contract <2 x double> [[COL_LOAD3]], [[SPLAT_SPLAT14]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd contract <2 x double> [[TMP3]], [[TMP2]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT17:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> undef, <2 x i32> <i32 3, i32 3>
-; CHECK-NEXT:    [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD5]], <2 x double> [[SPLAT_SPLAT17]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul contract <2 x double> [[COL_LOAD5]], [[SPLAT_SPLAT17]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd contract <2 x double> [[TMP5]], [[TMP4]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT20:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT20]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT20]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT23:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP5:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT23]], <2 x double> [[TMP4]])
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul contract <2 x double> [[COL_LOAD1]], [[SPLAT_SPLAT23]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fadd contract <2 x double> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT26:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> undef, <2 x i32> <i32 2, i32 2>
-; CHECK-NEXT:    [[TMP6:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD3]], <2 x double> [[SPLAT_SPLAT26]], <2 x double> [[TMP5]])
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul contract <2 x double> [[COL_LOAD3]], [[SPLAT_SPLAT26]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd contract <2 x double> [[TMP10]], [[TMP9]]
 ; CHECK-NEXT:    [[SPLAT_SPLAT29:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> undef, <2 x i32> <i32 3, i32 3>
-; CHECK-NEXT:    [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD5]], <2 x double> [[SPLAT_SPLAT29]], <2 x double> [[TMP6]])
-; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[C:%.*]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul contract <2 x double> [[COL_LOAD5]], [[SPLAT_SPLAT29]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fadd contract <2 x double> [[TMP12]], [[TMP11]]
+; CHECK-NEXT:    store <2 x double> [[TMP6]], ptr [[C:%.*]], align 8
 ; CHECK-NEXT:    [[VEC_GEP30:%.*]] = getelementptr double, ptr [[C]], i64 2
-; CHECK-NEXT:    store <2 x double> [[TMP7]], ptr [[VEC_GEP30]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP13]], ptr [[VEC_GEP30]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %a = load <8 x double>, ptr %A, align 8
   %b = load <8 x double>, ptr %B, align 8
 
   %c = call <4 x double> @llvm.matrix.multiply(<8 x double> %a, <8 x double> %b, i32 2, i32 4, i32 2)
 
   store <4 x double> %c, ptr %C, align 8
   ret void
 }
 
 declare <4 x double> @llvm.matrix.multiply(<8 x double>, <8 x double>, i32, i32, i32)