diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp --- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp +++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp @@ -312,7 +312,7 @@ default: return false; } - return isUniformShape(V) || isa(V); + return isUniformShape(V) || isa(V) || isa(V); } /// Propagate the shape information of instructions to their users. @@ -397,9 +397,71 @@ } } + /// Propagate the shape to operands of instructions with shape information. + void propagateShapeBackward() { + SmallVector WorkList; + // Worklist contains instruction for which we already know the shape. + for (auto &V : ShapeMap) + WorkList.push_back(V.first); + + // Pop an element with known shape. Traverse the operands, if their shape + // derives from the result shape and is unknown, add it and add them to the + // worklist. + LLVM_DEBUG(dbgs() << "Backward-propagate shapes:\n"); + while (!WorkList.empty()) { + Value *V = WorkList.back(); + WorkList.pop_back(); + + if (!isa(V)) + continue; + + Value *MatrixA; + Value *MatrixB; + Value *M; + Value *N; + Value *K; + if (match(V, m_Intrinsic( + m_Value(MatrixA), m_Value(MatrixB), m_Value(M), + m_Value(N), m_Value(K)))) { + if (setShapeInfo(MatrixA, {M, N})) + WorkList.push_back(MatrixA); + + if (setShapeInfo(MatrixB, {N, K})) + WorkList.push_back(MatrixB); + + } else if (match(V, m_Intrinsic( + m_Value(MatrixA), m_Value(M), m_Value(N)))) { + // Flip dimensions. + if (setShapeInfo(MatrixA, {M, N})) + WorkList.push_back(MatrixA); + } else if (match(V, m_Intrinsic( + m_Value(MatrixA), m_Value(), m_Value(), + m_Value(M), m_Value(N)))) { + if (setShapeInfo(MatrixA, {M, N})) { + WorkList.push_back(MatrixA); + } + } else if (isa(V) || + match(V, m_Intrinsic())) { + // Nothing to do, no matrix input. + } else if (isa(V)) { + // Nothing to do. We forward-propagated to this so we would just + // backward propagate to an instruction with an already known shape. + } else if (isUniformShape(V)) { + // Propagate to all operands. + ShapeInfo Shape = ShapeMap[V]; + for (Use &U : cast(V)->operands()) { + if (setShapeInfo(U.get(), Shape)) + WorkList.push_back(U.get()); + } + } + } + } + bool Visit() { - if (EnableShapePropagation) + if (EnableShapePropagation) { propagateShapeForward(); + propagateShapeBackward(); + } ReversePostOrderTraversal RPOT(&Func); bool Changed = false; @@ -414,6 +476,8 @@ Value *Op2; if (auto *BinOp = dyn_cast(&Inst)) Changed |= VisitBinaryOperator(BinOp); + if (match(&Inst, m_Load(m_Value(Op1)))) + Changed |= VisitLoad(&Inst, Op1, Builder); else if (match(&Inst, m_Store(m_Value(Op1), m_Value(Op2)))) Changed |= VisitStore(&Inst, Op1, Op2, Builder); } @@ -469,17 +533,10 @@ return true; } - /// Lowers llvm.matrix.columnwise.load. - /// - /// The intrinsic loads a matrix from memory using a stride between columns. - void LowerColumnwiseLoad(CallInst *Inst) { + void LowerLoad(Instruction *Inst, Value *Ptr, Value *Stride, ShapeInfo Shape) { IRBuilder<> Builder(Inst); - Value *Ptr = Inst->getArgOperand(0); - Value *Stride = Inst->getArgOperand(1); auto VType = cast(Inst->getType()); Value *EltPtr = createElementPtr(Ptr, VType->getElementType(), Builder); - ShapeInfo Shape(Inst->getArgOperand(2), Inst->getArgOperand(3)); - ColumnMatrixTy Result; // Distance between start of one column and the start of the next for (unsigned C = 0, E = Shape.NumColumns; C < E; ++C) { @@ -493,6 +550,15 @@ finalizeLowering(Inst, Result, Builder); } + /// Lowers llvm.matrix.columnwise.load. + /// + /// The intrinsic loads a matrix from memory using a stride between columns. + void LowerColumnwiseLoad(CallInst *Inst) { + Value *Ptr = Inst->getArgOperand(0); + Value *Stride = Inst->getArgOperand(1); + LowerLoad(Inst, Ptr, Stride, {Inst->getArgOperand(2), Inst->getArgOperand(3)}); + } + void LowerStore(Instruction *Inst, Value *Matrix, Value *Ptr, Value *Stride, ShapeInfo Shape) { IRBuilder<> Builder(Inst); @@ -673,6 +739,16 @@ finalizeLowering(Inst, Result, Builder); } + /// Lower load instructions, if shape information is available. + bool VisitLoad(Instruction *Inst, Value *Ptr, IRBuilder<> &Builder) { + auto I = ShapeMap.find(Inst); + if (I == ShapeMap.end()) + return false; + + LowerLoad(Inst, Ptr, Builder.getInt32(I->second.NumRows), I->second); + return true; + } + bool VisitStore(Instruction *Inst, Value *StoredVal, Value *Ptr, IRBuilder<> &Builder) { auto I = ShapeMap.find(StoredVal); diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll --- a/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll +++ b/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll @@ -7,232 +7,235 @@ ; CHECK-LABEL: @transpose_multiply( ; CHECK-NEXT: entry: -; Load input matrixes %A and %B. - -; CHECK-NEXT: [[A:%.*]] = load <9 x double>, <9 x double>* [[A_PTR:%.*]] -; CHECK-NEXT: [[B:%.*]] = load <9 x double>, <9 x double>* [[B_PTR:%.*]] - -; Extract columns from loaded value %A. - -; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> -; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> -; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> +; Load columns of input matrixes %A and %B. + +; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x double>* [[A_PTR:%.*]] to double* +; CHECK-NEXT: [[TMP1:%.*]] = bitcast double* [[TMP0]] to <3 x double>* +; CHECK-NEXT: [[TMP2:%.*]] = load <3 x double>, <3 x double>* [[TMP1]], align 8 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, double* [[TMP0]], i32 3 +; CHECK-NEXT: [[TMP4:%.*]] = bitcast double* [[TMP3]] to <3 x double>* +; CHECK-NEXT: [[TMP5:%.*]] = load <3 x double>, <3 x double>* [[TMP4]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr double, double* [[TMP0]], i32 6 +; CHECK-NEXT: [[TMP7:%.*]] = bitcast double* [[TMP6]] to <3 x double>* +; CHECK-NEXT: [[TMP8:%.*]] = load <3 x double>, <3 x double>* [[TMP7]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = bitcast <9 x double>* [[B_PTR:%.*]] to double* +; CHECK-NEXT: [[TMP10:%.*]] = bitcast double* [[TMP9]] to <3 x double>* +; CHECK-NEXT: [[TMP11:%.*]] = load <3 x double>, <3 x double>* [[TMP10]], align 8 +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr double, double* [[TMP9]], i32 3 +; CHECK-NEXT: [[TMP13:%.*]] = bitcast double* [[TMP12]] to <3 x double>* +; CHECK-NEXT: [[TMP14:%.*]] = load <3 x double>, <3 x double>* [[TMP13]], align 8 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr double, double* [[TMP9]], i32 6 +; CHECK-NEXT: [[TMP16:%.*]] = bitcast double* [[TMP15]] to <3 x double>* +; CHECK-NEXT: [[TMP17:%.*]] = load <3 x double>, <3 x double>* [[TMP16]], align 8 ; Transpose %A. -; CHECK-NEXT: [[TMP0:%.*]] = extractelement <3 x double> [[SPLIT]], i64 0 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <3 x double> undef, double [[TMP0]], i64 0 -; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 0 -; CHECK-NEXT: [[TMP3:%.*]] = insertelement <3 x double> [[TMP1]], double [[TMP2]], i64 1 -; CHECK-NEXT: [[TMP4:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 0 -; CHECK-NEXT: [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 2 -; CHECK-NEXT: [[TMP6:%.*]] = extractelement <3 x double> [[SPLIT]], i64 1 -; CHECK-NEXT: [[TMP7:%.*]] = insertelement <3 x double> undef, double [[TMP6]], i64 0 -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 1 -; CHECK-NEXT: [[TMP9:%.*]] = insertelement <3 x double> [[TMP7]], double [[TMP8]], i64 1 -; CHECK-NEXT: [[TMP10:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 1 -; CHECK-NEXT: [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 2 -; CHECK-NEXT: [[TMP12:%.*]] = extractelement <3 x double> [[SPLIT]], i64 2 -; CHECK-NEXT: [[TMP13:%.*]] = insertelement <3 x double> undef, double [[TMP12]], i64 0 -; CHECK-NEXT: [[TMP14:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 2 -; CHECK-NEXT: [[TMP15:%.*]] = insertelement <3 x double> [[TMP13]], double [[TMP14]], i64 1 -; CHECK-NEXT: [[TMP16:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 2 -; CHECK-NEXT: [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 2 - -; Extract columns from %B. - -; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> -; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> -; CHECK-NEXT: [[SPLIT5:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP18:%.*]] = extractelement <3 x double> [[TMP2]], i64 0 +; CHECK-NEXT: [[TMP19:%.*]] = insertelement <3 x double> undef, double [[TMP18]], i64 0 +; CHECK-NEXT: [[TMP20:%.*]] = extractelement <3 x double> [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP21:%.*]] = insertelement <3 x double> [[TMP19]], double [[TMP20]], i64 1 +; CHECK-NEXT: [[TMP22:%.*]] = extractelement <3 x double> [[TMP8]], i64 0 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <3 x double> [[TMP21]], double [[TMP22]], i64 2 +; CHECK-NEXT: [[TMP24:%.*]] = extractelement <3 x double> [[TMP2]], i64 1 +; CHECK-NEXT: [[TMP25:%.*]] = insertelement <3 x double> undef, double [[TMP24]], i64 0 +; CHECK-NEXT: [[TMP26:%.*]] = extractelement <3 x double> [[TMP5]], i64 1 +; CHECK-NEXT: [[TMP27:%.*]] = insertelement <3 x double> [[TMP25]], double [[TMP26]], i64 1 +; CHECK-NEXT: [[TMP28:%.*]] = extractelement <3 x double> [[TMP8]], i64 1 +; CHECK-NEXT: [[TMP29:%.*]] = insertelement <3 x double> [[TMP27]], double [[TMP28]], i64 2 +; CHECK-NEXT: [[TMP30:%.*]] = extractelement <3 x double> [[TMP2]], i64 2 +; CHECK-NEXT: [[TMP31:%.*]] = insertelement <3 x double> undef, double [[TMP30]], i64 0 +; CHECK-NEXT: [[TMP32:%.*]] = extractelement <3 x double> [[TMP5]], i64 2 +; CHECK-NEXT: [[TMP33:%.*]] = insertelement <3 x double> [[TMP31]], double [[TMP32]], i64 1 +; CHECK-NEXT: [[TMP34:%.*]] = extractelement <3 x double> [[TMP8]], i64 2 +; CHECK-NEXT: [[TMP35:%.*]] = insertelement <3 x double> [[TMP33]], double [[TMP34]], i64 2 +; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP36:%.*]] = extractelement <3 x double> [[TMP11]], i64 0 ; Lower multiply(transpose(%A), %B) -; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP18:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP18]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP36]], i32 0 ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP19:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]] -; CHECK-NEXT: [[BLOCK6:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP20:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x double> undef, double [[TMP20]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT7]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP21:%.*]] = fmul <1 x double> [[BLOCK6]], [[SPLAT_SPLAT8]] -; CHECK-NEXT: [[TMP22:%.*]] = fadd <1 x double> [[TMP19]], [[TMP21]] -; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP23:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> undef, double [[TMP23]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP24:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]] -; CHECK-NEXT: [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]] -; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP26]], <3 x i32> -; CHECK-NEXT: [[BLOCK12:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP28:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT13:%.*]] = insertelement <1 x double> undef, double [[TMP28]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP29:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]] -; CHECK-NEXT: [[BLOCK15:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP30:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT16:%.*]] = insertelement <1 x double> undef, double [[TMP30]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT16]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP31:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]] -; CHECK-NEXT: [[TMP32:%.*]] = fadd <1 x double> [[TMP29]], [[TMP31]] -; CHECK-NEXT: [[BLOCK18:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP33:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT19:%.*]] = insertelement <1 x double> undef, double [[TMP33]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT19]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP34:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]] -; CHECK-NEXT: [[TMP35:%.*]] = fadd <1 x double> [[TMP32]], [[TMP34]] -; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <1 x double> [[TMP35]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP37:%.*]] = shufflevector <3 x double> [[TMP27]], <3 x double> [[TMP36]], <3 x i32> -; CHECK-NEXT: [[BLOCK21:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP38:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT22:%.*]] = insertelement <1 x double> undef, double [[TMP38]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP39:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]] -; CHECK-NEXT: [[BLOCK24:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP40:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT25:%.*]] = insertelement <1 x double> undef, double [[TMP40]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT26:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT25]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP41:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]] -; CHECK-NEXT: [[TMP42:%.*]] = fadd <1 x double> [[TMP39]], [[TMP41]] -; CHECK-NEXT: [[BLOCK27:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP43:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT28:%.*]] = insertelement <1 x double> undef, double [[TMP43]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT29:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT28]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP44:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]] -; CHECK-NEXT: [[TMP45:%.*]] = fadd <1 x double> [[TMP42]], [[TMP44]] -; CHECK-NEXT: [[TMP46:%.*]] = shufflevector <1 x double> [[TMP45]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP47:%.*]] = shufflevector <3 x double> [[TMP37]], <3 x double> [[TMP46]], <3 x i32> -; CHECK-NEXT: [[BLOCK30:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP48:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT31:%.*]] = insertelement <1 x double> undef, double [[TMP48]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP49:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]] -; CHECK-NEXT: [[BLOCK33:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP50:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT34:%.*]] = insertelement <1 x double> undef, double [[TMP50]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT35:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT34]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP51:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]] -; CHECK-NEXT: [[TMP52:%.*]] = fadd <1 x double> [[TMP49]], [[TMP51]] -; CHECK-NEXT: [[BLOCK36:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP53:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT37:%.*]] = insertelement <1 x double> undef, double [[TMP53]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT38:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT37]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP54:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]] -; CHECK-NEXT: [[TMP55:%.*]] = fadd <1 x double> [[TMP52]], [[TMP54]] -; CHECK-NEXT: [[TMP56:%.*]] = shufflevector <1 x double> [[TMP55]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP57:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP56]], <3 x i32> -; CHECK-NEXT: [[BLOCK39:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP58:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT40:%.*]] = insertelement <1 x double> undef, double [[TMP58]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP59:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]] -; CHECK-NEXT: [[BLOCK42:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP60:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT43:%.*]] = insertelement <1 x double> undef, double [[TMP60]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT44:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT43]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP61:%.*]] = fmul <1 x double> [[BLOCK42]], [[SPLAT_SPLAT44]] -; CHECK-NEXT: [[TMP62:%.*]] = fadd <1 x double> [[TMP59]], [[TMP61]] -; CHECK-NEXT: [[BLOCK45:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP63:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT46:%.*]] = insertelement <1 x double> undef, double [[TMP63]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT47:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT46]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP64:%.*]] = fmul <1 x double> [[BLOCK45]], [[SPLAT_SPLAT47]] -; CHECK-NEXT: [[TMP65:%.*]] = fadd <1 x double> [[TMP62]], [[TMP64]] -; CHECK-NEXT: [[TMP66:%.*]] = shufflevector <1 x double> [[TMP65]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP67:%.*]] = shufflevector <3 x double> [[TMP57]], <3 x double> [[TMP66]], <3 x i32> -; CHECK-NEXT: [[BLOCK48:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP68:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT49:%.*]] = insertelement <1 x double> undef, double [[TMP68]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP69:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]] -; CHECK-NEXT: [[BLOCK51:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP70:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT52:%.*]] = insertelement <1 x double> undef, double [[TMP70]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT53:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT52]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP71:%.*]] = fmul <1 x double> [[BLOCK51]], [[SPLAT_SPLAT53]] -; CHECK-NEXT: [[TMP72:%.*]] = fadd <1 x double> [[TMP69]], [[TMP71]] -; CHECK-NEXT: [[BLOCK54:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP73:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT55:%.*]] = insertelement <1 x double> undef, double [[TMP73]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT56:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT55]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP74:%.*]] = fmul <1 x double> [[BLOCK54]], [[SPLAT_SPLAT56]] -; CHECK-NEXT: [[TMP75:%.*]] = fadd <1 x double> [[TMP72]], [[TMP74]] -; CHECK-NEXT: [[TMP76:%.*]] = shufflevector <1 x double> [[TMP75]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP77:%.*]] = shufflevector <3 x double> [[TMP67]], <3 x double> [[TMP76]], <3 x i32> -; CHECK-NEXT: [[BLOCK57:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP78:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT58:%.*]] = insertelement <1 x double> undef, double [[TMP78]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP79:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]] -; CHECK-NEXT: [[BLOCK60:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP80:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT61:%.*]] = insertelement <1 x double> undef, double [[TMP80]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT62:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT61]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP81:%.*]] = fmul <1 x double> [[BLOCK60]], [[SPLAT_SPLAT62]] -; CHECK-NEXT: [[TMP82:%.*]] = fadd <1 x double> [[TMP79]], [[TMP81]] -; CHECK-NEXT: [[BLOCK63:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP83:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT64:%.*]] = insertelement <1 x double> undef, double [[TMP83]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT65:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT64]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP84:%.*]] = fmul <1 x double> [[BLOCK63]], [[SPLAT_SPLAT65]] -; CHECK-NEXT: [[TMP85:%.*]] = fadd <1 x double> [[TMP82]], [[TMP84]] -; CHECK-NEXT: [[TMP86:%.*]] = shufflevector <1 x double> [[TMP85]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP87:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP86]], <3 x i32> -; CHECK-NEXT: [[BLOCK66:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP88:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT67:%.*]] = insertelement <1 x double> undef, double [[TMP88]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP89:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]] -; CHECK-NEXT: [[BLOCK69:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP90:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT70:%.*]] = insertelement <1 x double> undef, double [[TMP90]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT71:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT70]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP91:%.*]] = fmul <1 x double> [[BLOCK69]], [[SPLAT_SPLAT71]] -; CHECK-NEXT: [[TMP92:%.*]] = fadd <1 x double> [[TMP89]], [[TMP91]] -; CHECK-NEXT: [[BLOCK72:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP93:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT73:%.*]] = insertelement <1 x double> undef, double [[TMP93]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT74:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT73]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP94:%.*]] = fmul <1 x double> [[BLOCK72]], [[SPLAT_SPLAT74]] -; CHECK-NEXT: [[TMP95:%.*]] = fadd <1 x double> [[TMP92]], [[TMP94]] -; CHECK-NEXT: [[TMP96:%.*]] = shufflevector <1 x double> [[TMP95]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP97:%.*]] = shufflevector <3 x double> [[TMP87]], <3 x double> [[TMP96]], <3 x i32> -; CHECK-NEXT: [[BLOCK75:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP98:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT76:%.*]] = insertelement <1 x double> undef, double [[TMP98]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP99:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]] -; CHECK-NEXT: [[BLOCK78:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP100:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT79:%.*]] = insertelement <1 x double> undef, double [[TMP100]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT80:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT79]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP101:%.*]] = fmul <1 x double> [[BLOCK78]], [[SPLAT_SPLAT80]] -; CHECK-NEXT: [[TMP102:%.*]] = fadd <1 x double> [[TMP99]], [[TMP101]] -; CHECK-NEXT: [[BLOCK81:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP103:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT82:%.*]] = insertelement <1 x double> undef, double [[TMP103]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT83:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT82]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP104:%.*]] = fmul <1 x double> [[BLOCK81]], [[SPLAT_SPLAT83]] -; CHECK-NEXT: [[TMP105:%.*]] = fadd <1 x double> [[TMP102]], [[TMP104]] -; CHECK-NEXT: [[TMP106:%.*]] = shufflevector <1 x double> [[TMP105]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP107:%.*]] = shufflevector <3 x double> [[TMP97]], <3 x double> [[TMP106]], <3 x i32> +; CHECK-NEXT: [[TMP37:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]] +; CHECK-NEXT: [[BLOCK1:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP38:%.*]] = extractelement <3 x double> [[TMP11]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT2:%.*]] = insertelement <1 x double> undef, double [[TMP38]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT3:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT2]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP39:%.*]] = fmul <1 x double> [[BLOCK1]], [[SPLAT_SPLAT3]] +; CHECK-NEXT: [[TMP40:%.*]] = fadd <1 x double> [[TMP37]], [[TMP39]] +; CHECK-NEXT: [[BLOCK4:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP41:%.*]] = extractelement <3 x double> [[TMP11]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x double> undef, double [[TMP41]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT5]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP42:%.*]] = fmul <1 x double> [[BLOCK4]], [[SPLAT_SPLAT6]] +; CHECK-NEXT: [[TMP43:%.*]] = fadd <1 x double> [[TMP40]], [[TMP42]] +; CHECK-NEXT: [[TMP44:%.*]] = shufflevector <1 x double> [[TMP43]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP45:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP44]], <3 x i32> +; CHECK-NEXT: [[BLOCK7:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP46:%.*]] = extractelement <3 x double> [[TMP11]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT8:%.*]] = insertelement <1 x double> undef, double [[TMP46]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT9:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT8]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP47:%.*]] = fmul <1 x double> [[BLOCK7]], [[SPLAT_SPLAT9]] +; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP48:%.*]] = extractelement <3 x double> [[TMP11]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x double> undef, double [[TMP48]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT11]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP49:%.*]] = fmul <1 x double> [[BLOCK10]], [[SPLAT_SPLAT12]] +; CHECK-NEXT: [[TMP50:%.*]] = fadd <1 x double> [[TMP47]], [[TMP49]] +; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP51:%.*]] = extractelement <3 x double> [[TMP11]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT14:%.*]] = insertelement <1 x double> undef, double [[TMP51]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT14]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP52:%.*]] = fmul <1 x double> [[BLOCK13]], [[SPLAT_SPLAT15]] +; CHECK-NEXT: [[TMP53:%.*]] = fadd <1 x double> [[TMP50]], [[TMP52]] +; CHECK-NEXT: [[TMP54:%.*]] = shufflevector <1 x double> [[TMP53]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP55:%.*]] = shufflevector <3 x double> [[TMP45]], <3 x double> [[TMP54]], <3 x i32> +; CHECK-NEXT: [[BLOCK16:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP56:%.*]] = extractelement <3 x double> [[TMP11]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x double> undef, double [[TMP56]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT17]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP57:%.*]] = fmul <1 x double> [[BLOCK16]], [[SPLAT_SPLAT18]] +; CHECK-NEXT: [[BLOCK19:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP58:%.*]] = extractelement <3 x double> [[TMP11]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT20:%.*]] = insertelement <1 x double> undef, double [[TMP58]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT21:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT20]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP59:%.*]] = fmul <1 x double> [[BLOCK19]], [[SPLAT_SPLAT21]] +; CHECK-NEXT: [[TMP60:%.*]] = fadd <1 x double> [[TMP57]], [[TMP59]] +; CHECK-NEXT: [[BLOCK22:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP61:%.*]] = extractelement <3 x double> [[TMP11]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x double> undef, double [[TMP61]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT23]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP62:%.*]] = fmul <1 x double> [[BLOCK22]], [[SPLAT_SPLAT24]] +; CHECK-NEXT: [[TMP63:%.*]] = fadd <1 x double> [[TMP60]], [[TMP62]] +; CHECK-NEXT: [[TMP64:%.*]] = shufflevector <1 x double> [[TMP63]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP65:%.*]] = shufflevector <3 x double> [[TMP55]], <3 x double> [[TMP64]], <3 x i32> +; CHECK-NEXT: [[BLOCK25:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP66:%.*]] = extractelement <3 x double> [[TMP14]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT26:%.*]] = insertelement <1 x double> undef, double [[TMP66]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT27:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT26]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP67:%.*]] = fmul <1 x double> [[BLOCK25]], [[SPLAT_SPLAT27]] +; CHECK-NEXT: [[BLOCK28:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP68:%.*]] = extractelement <3 x double> [[TMP14]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT29:%.*]] = insertelement <1 x double> undef, double [[TMP68]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT30:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT29]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP69:%.*]] = fmul <1 x double> [[BLOCK28]], [[SPLAT_SPLAT30]] +; CHECK-NEXT: [[TMP70:%.*]] = fadd <1 x double> [[TMP67]], [[TMP69]] +; CHECK-NEXT: [[BLOCK31:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP71:%.*]] = extractelement <3 x double> [[TMP14]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT32:%.*]] = insertelement <1 x double> undef, double [[TMP71]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT33:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT32]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP72:%.*]] = fmul <1 x double> [[BLOCK31]], [[SPLAT_SPLAT33]] +; CHECK-NEXT: [[TMP73:%.*]] = fadd <1 x double> [[TMP70]], [[TMP72]] +; CHECK-NEXT: [[TMP74:%.*]] = shufflevector <1 x double> [[TMP73]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP75:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP74]], <3 x i32> +; CHECK-NEXT: [[BLOCK34:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP76:%.*]] = extractelement <3 x double> [[TMP14]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT35:%.*]] = insertelement <1 x double> undef, double [[TMP76]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT36:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT35]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP77:%.*]] = fmul <1 x double> [[BLOCK34]], [[SPLAT_SPLAT36]] +; CHECK-NEXT: [[BLOCK37:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP78:%.*]] = extractelement <3 x double> [[TMP14]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT38:%.*]] = insertelement <1 x double> undef, double [[TMP78]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT39:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT38]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP79:%.*]] = fmul <1 x double> [[BLOCK37]], [[SPLAT_SPLAT39]] +; CHECK-NEXT: [[TMP80:%.*]] = fadd <1 x double> [[TMP77]], [[TMP79]] +; CHECK-NEXT: [[BLOCK40:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP81:%.*]] = extractelement <3 x double> [[TMP14]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT41:%.*]] = insertelement <1 x double> undef, double [[TMP81]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT42:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT41]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP82:%.*]] = fmul <1 x double> [[BLOCK40]], [[SPLAT_SPLAT42]] +; CHECK-NEXT: [[TMP83:%.*]] = fadd <1 x double> [[TMP80]], [[TMP82]] +; CHECK-NEXT: [[TMP84:%.*]] = shufflevector <1 x double> [[TMP83]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP85:%.*]] = shufflevector <3 x double> [[TMP75]], <3 x double> [[TMP84]], <3 x i32> +; CHECK-NEXT: [[BLOCK43:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP86:%.*]] = extractelement <3 x double> [[TMP14]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT44:%.*]] = insertelement <1 x double> undef, double [[TMP86]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT45:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT44]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP87:%.*]] = fmul <1 x double> [[BLOCK43]], [[SPLAT_SPLAT45]] +; CHECK-NEXT: [[BLOCK46:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP88:%.*]] = extractelement <3 x double> [[TMP14]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT47:%.*]] = insertelement <1 x double> undef, double [[TMP88]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT48:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT47]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP89:%.*]] = fmul <1 x double> [[BLOCK46]], [[SPLAT_SPLAT48]] +; CHECK-NEXT: [[TMP90:%.*]] = fadd <1 x double> [[TMP87]], [[TMP89]] +; CHECK-NEXT: [[BLOCK49:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP91:%.*]] = extractelement <3 x double> [[TMP14]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT50:%.*]] = insertelement <1 x double> undef, double [[TMP91]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT51:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT50]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP92:%.*]] = fmul <1 x double> [[BLOCK49]], [[SPLAT_SPLAT51]] +; CHECK-NEXT: [[TMP93:%.*]] = fadd <1 x double> [[TMP90]], [[TMP92]] +; CHECK-NEXT: [[TMP94:%.*]] = shufflevector <1 x double> [[TMP93]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP95:%.*]] = shufflevector <3 x double> [[TMP85]], <3 x double> [[TMP94]], <3 x i32> +; CHECK-NEXT: [[BLOCK52:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP96:%.*]] = extractelement <3 x double> [[TMP17]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT53:%.*]] = insertelement <1 x double> undef, double [[TMP96]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT54:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT53]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP97:%.*]] = fmul <1 x double> [[BLOCK52]], [[SPLAT_SPLAT54]] +; CHECK-NEXT: [[BLOCK55:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP98:%.*]] = extractelement <3 x double> [[TMP17]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT56:%.*]] = insertelement <1 x double> undef, double [[TMP98]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT57:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT56]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP99:%.*]] = fmul <1 x double> [[BLOCK55]], [[SPLAT_SPLAT57]] +; CHECK-NEXT: [[TMP100:%.*]] = fadd <1 x double> [[TMP97]], [[TMP99]] +; CHECK-NEXT: [[BLOCK58:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP101:%.*]] = extractelement <3 x double> [[TMP17]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT59:%.*]] = insertelement <1 x double> undef, double [[TMP101]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT60:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT59]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP102:%.*]] = fmul <1 x double> [[BLOCK58]], [[SPLAT_SPLAT60]] +; CHECK-NEXT: [[TMP103:%.*]] = fadd <1 x double> [[TMP100]], [[TMP102]] +; CHECK-NEXT: [[TMP104:%.*]] = shufflevector <1 x double> [[TMP103]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP105:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP104]], <3 x i32> +; CHECK-NEXT: [[BLOCK61:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP106:%.*]] = extractelement <3 x double> [[TMP17]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT62:%.*]] = insertelement <1 x double> undef, double [[TMP106]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT63:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT62]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP107:%.*]] = fmul <1 x double> [[BLOCK61]], [[SPLAT_SPLAT63]] +; CHECK-NEXT: [[BLOCK64:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP108:%.*]] = extractelement <3 x double> [[TMP17]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT65:%.*]] = insertelement <1 x double> undef, double [[TMP108]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT66:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT65]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP109:%.*]] = fmul <1 x double> [[BLOCK64]], [[SPLAT_SPLAT66]] +; CHECK-NEXT: [[TMP110:%.*]] = fadd <1 x double> [[TMP107]], [[TMP109]] +; CHECK-NEXT: [[BLOCK67:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP111:%.*]] = extractelement <3 x double> [[TMP17]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT68:%.*]] = insertelement <1 x double> undef, double [[TMP111]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT69:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT68]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP112:%.*]] = fmul <1 x double> [[BLOCK67]], [[SPLAT_SPLAT69]] +; CHECK-NEXT: [[TMP113:%.*]] = fadd <1 x double> [[TMP110]], [[TMP112]] +; CHECK-NEXT: [[TMP114:%.*]] = shufflevector <1 x double> [[TMP113]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP115:%.*]] = shufflevector <3 x double> [[TMP105]], <3 x double> [[TMP114]], <3 x i32> +; CHECK-NEXT: [[BLOCK70:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP116:%.*]] = extractelement <3 x double> [[TMP17]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT71:%.*]] = insertelement <1 x double> undef, double [[TMP116]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT72:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT71]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP117:%.*]] = fmul <1 x double> [[BLOCK70]], [[SPLAT_SPLAT72]] +; CHECK-NEXT: [[BLOCK73:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP118:%.*]] = extractelement <3 x double> [[TMP17]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT74:%.*]] = insertelement <1 x double> undef, double [[TMP118]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT75:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT74]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP119:%.*]] = fmul <1 x double> [[BLOCK73]], [[SPLAT_SPLAT75]] +; CHECK-NEXT: [[TMP120:%.*]] = fadd <1 x double> [[TMP117]], [[TMP119]] +; CHECK-NEXT: [[BLOCK76:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP121:%.*]] = extractelement <3 x double> [[TMP17]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT77:%.*]] = insertelement <1 x double> undef, double [[TMP121]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT78:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT77]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP122:%.*]] = fmul <1 x double> [[BLOCK76]], [[SPLAT_SPLAT78]] +; CHECK-NEXT: [[TMP123:%.*]] = fadd <1 x double> [[TMP120]], [[TMP122]] +; CHECK-NEXT: [[TMP124:%.*]] = shufflevector <1 x double> [[TMP123]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP125:%.*]] = shufflevector <3 x double> [[TMP115]], <3 x double> [[TMP124]], <3 x i32> ; Store result columns. -; CHECK-NEXT: [[TMP108:%.*]] = bitcast <9 x double>* [[C_PTR:%.*]] to double* -; CHECK-NEXT: [[TMP109:%.*]] = bitcast double* [[TMP108]] to <3 x double>* -; CHECK-NEXT: store <3 x double> [[TMP47]], <3 x double>* [[TMP109]], align 8 -; CHECK-NEXT: [[TMP110:%.*]] = getelementptr double, double* [[TMP108]], i32 3 -; CHECK-NEXT: [[TMP111:%.*]] = bitcast double* [[TMP110]] to <3 x double>* -; CHECK-NEXT: store <3 x double> [[TMP77]], <3 x double>* [[TMP111]], align 8 -; CHECK-NEXT: [[TMP112:%.*]] = getelementptr double, double* [[TMP108]], i32 6 -; CHECK-NEXT: [[TMP113:%.*]] = bitcast double* [[TMP112]] to <3 x double>* -; CHECK-NEXT: store <3 x double> [[TMP107]], <3 x double>* [[TMP113]], align 8 +; CHECK-NEXT: [[TMP126:%.*]] = bitcast <9 x double>* [[C_PTR:%.*]] to double* +; CHECK-NEXT: [[TMP127:%.*]] = bitcast double* [[TMP126]] to <3 x double>* +; CHECK-NEXT: store <3 x double> [[TMP65]], <3 x double>* [[TMP127]], align 8 +; CHECK-NEXT: [[TMP128:%.*]] = getelementptr double, double* [[TMP126]], i32 3 +; CHECK-NEXT: [[TMP129:%.*]] = bitcast double* [[TMP128]] to <3 x double>* +; CHECK-NEXT: store <3 x double> [[TMP95]], <3 x double>* [[TMP129]], align 8 +; CHECK-NEXT: [[TMP130:%.*]] = getelementptr double, double* [[TMP126]], i32 6 +; CHECK-NEXT: [[TMP131:%.*]] = bitcast double* [[TMP130]] to <3 x double>* +; CHECK-NEXT: store <3 x double> [[TMP125]], <3 x double>* [[TMP131]], align 8 ; CHECK-NEXT: ret void ; - entry: %a = load <9 x double>, <9 x double>* %A.Ptr %b = load <9 x double>, <9 x double>* %B.Ptr @@ -249,248 +252,251 @@ ; CHECK-LABEL: @transpose_multiply_add( ; CHECK-NEXT: entry: -; Load input matrixes %A and %B. - -; CHECK-NEXT: [[A:%.*]] = load <9 x double>, <9 x double>* [[A_PTR:%.*]] -; CHECK-NEXT: [[B:%.*]] = load <9 x double>, <9 x double>* [[B_PTR:%.*]] - -; Extract columns from loaded value %A. - -; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> -; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> -; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> +; Load columns of input matrixes %A and %B. + +; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x double>* [[A_PTR:%.*]] to double* +; CHECK-NEXT: [[TMP1:%.*]] = bitcast double* [[TMP0]] to <3 x double>* +; CHECK-NEXT: [[TMP2:%.*]] = load <3 x double>, <3 x double>* [[TMP1]], align 8 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, double* [[TMP0]], i32 3 +; CHECK-NEXT: [[TMP4:%.*]] = bitcast double* [[TMP3]] to <3 x double>* +; CHECK-NEXT: [[TMP5:%.*]] = load <3 x double>, <3 x double>* [[TMP4]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr double, double* [[TMP0]], i32 6 +; CHECK-NEXT: [[TMP7:%.*]] = bitcast double* [[TMP6]] to <3 x double>* +; CHECK-NEXT: [[TMP8:%.*]] = load <3 x double>, <3 x double>* [[TMP7]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = bitcast <9 x double>* [[B_PTR:%.*]] to double* +; CHECK-NEXT: [[TMP10:%.*]] = bitcast double* [[TMP9]] to <3 x double>* +; CHECK-NEXT: [[TMP11:%.*]] = load <3 x double>, <3 x double>* [[TMP10]], align 8 +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr double, double* [[TMP9]], i32 3 +; CHECK-NEXT: [[TMP13:%.*]] = bitcast double* [[TMP12]] to <3 x double>* +; CHECK-NEXT: [[TMP14:%.*]] = load <3 x double>, <3 x double>* [[TMP13]], align 8 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr double, double* [[TMP9]], i32 6 +; CHECK-NEXT: [[TMP16:%.*]] = bitcast double* [[TMP15]] to <3 x double>* +; CHECK-NEXT: [[TMP17:%.*]] = load <3 x double>, <3 x double>* [[TMP16]], align 8 ; Transpose %A. -; CHECK-NEXT: [[TMP0:%.*]] = extractelement <3 x double> [[SPLIT]], i64 0 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <3 x double> undef, double [[TMP0]], i64 0 -; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 0 -; CHECK-NEXT: [[TMP3:%.*]] = insertelement <3 x double> [[TMP1]], double [[TMP2]], i64 1 -; CHECK-NEXT: [[TMP4:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 0 -; CHECK-NEXT: [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 2 -; CHECK-NEXT: [[TMP6:%.*]] = extractelement <3 x double> [[SPLIT]], i64 1 -; CHECK-NEXT: [[TMP7:%.*]] = insertelement <3 x double> undef, double [[TMP6]], i64 0 -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 1 -; CHECK-NEXT: [[TMP9:%.*]] = insertelement <3 x double> [[TMP7]], double [[TMP8]], i64 1 -; CHECK-NEXT: [[TMP10:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 1 -; CHECK-NEXT: [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 2 -; CHECK-NEXT: [[TMP12:%.*]] = extractelement <3 x double> [[SPLIT]], i64 2 -; CHECK-NEXT: [[TMP13:%.*]] = insertelement <3 x double> undef, double [[TMP12]], i64 0 -; CHECK-NEXT: [[TMP14:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 2 -; CHECK-NEXT: [[TMP15:%.*]] = insertelement <3 x double> [[TMP13]], double [[TMP14]], i64 1 -; CHECK-NEXT: [[TMP16:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 2 -; CHECK-NEXT: [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 2 - -; Extract columns from %B. - -; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> -; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> -; CHECK-NEXT: [[SPLIT5:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP18:%.*]] = extractelement <3 x double> [[TMP2]], i64 0 +; CHECK-NEXT: [[TMP19:%.*]] = insertelement <3 x double> undef, double [[TMP18]], i64 0 +; CHECK-NEXT: [[TMP20:%.*]] = extractelement <3 x double> [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP21:%.*]] = insertelement <3 x double> [[TMP19]], double [[TMP20]], i64 1 +; CHECK-NEXT: [[TMP22:%.*]] = extractelement <3 x double> [[TMP8]], i64 0 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <3 x double> [[TMP21]], double [[TMP22]], i64 2 +; CHECK-NEXT: [[TMP24:%.*]] = extractelement <3 x double> [[TMP2]], i64 1 +; CHECK-NEXT: [[TMP25:%.*]] = insertelement <3 x double> undef, double [[TMP24]], i64 0 +; CHECK-NEXT: [[TMP26:%.*]] = extractelement <3 x double> [[TMP5]], i64 1 +; CHECK-NEXT: [[TMP27:%.*]] = insertelement <3 x double> [[TMP25]], double [[TMP26]], i64 1 +; CHECK-NEXT: [[TMP28:%.*]] = extractelement <3 x double> [[TMP8]], i64 1 +; CHECK-NEXT: [[TMP29:%.*]] = insertelement <3 x double> [[TMP27]], double [[TMP28]], i64 2 +; CHECK-NEXT: [[TMP30:%.*]] = extractelement <3 x double> [[TMP2]], i64 2 +; CHECK-NEXT: [[TMP31:%.*]] = insertelement <3 x double> undef, double [[TMP30]], i64 0 +; CHECK-NEXT: [[TMP32:%.*]] = extractelement <3 x double> [[TMP5]], i64 2 +; CHECK-NEXT: [[TMP33:%.*]] = insertelement <3 x double> [[TMP31]], double [[TMP32]], i64 1 +; CHECK-NEXT: [[TMP34:%.*]] = extractelement <3 x double> [[TMP8]], i64 2 +; CHECK-NEXT: [[TMP35:%.*]] = insertelement <3 x double> [[TMP33]], double [[TMP34]], i64 2 +; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP36:%.*]] = extractelement <3 x double> [[TMP11]], i64 0 ; Lower multiply(transpose(%A), %B) -; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP18:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP18]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP36]], i32 0 ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP19:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]] -; CHECK-NEXT: [[BLOCK6:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP20:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x double> undef, double [[TMP20]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT7]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP21:%.*]] = fmul <1 x double> [[BLOCK6]], [[SPLAT_SPLAT8]] -; CHECK-NEXT: [[TMP22:%.*]] = fadd <1 x double> [[TMP19]], [[TMP21]] -; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP23:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> undef, double [[TMP23]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP24:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]] -; CHECK-NEXT: [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]] -; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP26]], <3 x i32> -; CHECK-NEXT: [[BLOCK12:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP28:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT13:%.*]] = insertelement <1 x double> undef, double [[TMP28]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP29:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]] -; CHECK-NEXT: [[BLOCK15:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP30:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT16:%.*]] = insertelement <1 x double> undef, double [[TMP30]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT16]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP31:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]] -; CHECK-NEXT: [[TMP32:%.*]] = fadd <1 x double> [[TMP29]], [[TMP31]] -; CHECK-NEXT: [[BLOCK18:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP33:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT19:%.*]] = insertelement <1 x double> undef, double [[TMP33]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT19]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP34:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]] -; CHECK-NEXT: [[TMP35:%.*]] = fadd <1 x double> [[TMP32]], [[TMP34]] -; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <1 x double> [[TMP35]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP37:%.*]] = shufflevector <3 x double> [[TMP27]], <3 x double> [[TMP36]], <3 x i32> -; CHECK-NEXT: [[BLOCK21:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP38:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT22:%.*]] = insertelement <1 x double> undef, double [[TMP38]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP39:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]] -; CHECK-NEXT: [[BLOCK24:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP40:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT25:%.*]] = insertelement <1 x double> undef, double [[TMP40]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT26:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT25]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP41:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]] -; CHECK-NEXT: [[TMP42:%.*]] = fadd <1 x double> [[TMP39]], [[TMP41]] -; CHECK-NEXT: [[BLOCK27:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP43:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT28:%.*]] = insertelement <1 x double> undef, double [[TMP43]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT29:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT28]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP44:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]] -; CHECK-NEXT: [[TMP45:%.*]] = fadd <1 x double> [[TMP42]], [[TMP44]] -; CHECK-NEXT: [[TMP46:%.*]] = shufflevector <1 x double> [[TMP45]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP47:%.*]] = shufflevector <3 x double> [[TMP37]], <3 x double> [[TMP46]], <3 x i32> -; CHECK-NEXT: [[BLOCK30:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP48:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT31:%.*]] = insertelement <1 x double> undef, double [[TMP48]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP49:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]] -; CHECK-NEXT: [[BLOCK33:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP50:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT34:%.*]] = insertelement <1 x double> undef, double [[TMP50]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT35:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT34]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP51:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]] -; CHECK-NEXT: [[TMP52:%.*]] = fadd <1 x double> [[TMP49]], [[TMP51]] -; CHECK-NEXT: [[BLOCK36:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP53:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT37:%.*]] = insertelement <1 x double> undef, double [[TMP53]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT38:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT37]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP54:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]] -; CHECK-NEXT: [[TMP55:%.*]] = fadd <1 x double> [[TMP52]], [[TMP54]] -; CHECK-NEXT: [[TMP56:%.*]] = shufflevector <1 x double> [[TMP55]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP57:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP56]], <3 x i32> -; CHECK-NEXT: [[BLOCK39:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP58:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT40:%.*]] = insertelement <1 x double> undef, double [[TMP58]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP59:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]] -; CHECK-NEXT: [[BLOCK42:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP60:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT43:%.*]] = insertelement <1 x double> undef, double [[TMP60]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT44:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT43]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP61:%.*]] = fmul <1 x double> [[BLOCK42]], [[SPLAT_SPLAT44]] -; CHECK-NEXT: [[TMP62:%.*]] = fadd <1 x double> [[TMP59]], [[TMP61]] -; CHECK-NEXT: [[BLOCK45:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP63:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT46:%.*]] = insertelement <1 x double> undef, double [[TMP63]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT47:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT46]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP64:%.*]] = fmul <1 x double> [[BLOCK45]], [[SPLAT_SPLAT47]] -; CHECK-NEXT: [[TMP65:%.*]] = fadd <1 x double> [[TMP62]], [[TMP64]] -; CHECK-NEXT: [[TMP66:%.*]] = shufflevector <1 x double> [[TMP65]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP67:%.*]] = shufflevector <3 x double> [[TMP57]], <3 x double> [[TMP66]], <3 x i32> -; CHECK-NEXT: [[BLOCK48:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP68:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT49:%.*]] = insertelement <1 x double> undef, double [[TMP68]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP69:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]] -; CHECK-NEXT: [[BLOCK51:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP70:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT52:%.*]] = insertelement <1 x double> undef, double [[TMP70]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT53:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT52]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP71:%.*]] = fmul <1 x double> [[BLOCK51]], [[SPLAT_SPLAT53]] -; CHECK-NEXT: [[TMP72:%.*]] = fadd <1 x double> [[TMP69]], [[TMP71]] -; CHECK-NEXT: [[BLOCK54:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP73:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT55:%.*]] = insertelement <1 x double> undef, double [[TMP73]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT56:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT55]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP74:%.*]] = fmul <1 x double> [[BLOCK54]], [[SPLAT_SPLAT56]] -; CHECK-NEXT: [[TMP75:%.*]] = fadd <1 x double> [[TMP72]], [[TMP74]] -; CHECK-NEXT: [[TMP76:%.*]] = shufflevector <1 x double> [[TMP75]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP77:%.*]] = shufflevector <3 x double> [[TMP67]], <3 x double> [[TMP76]], <3 x i32> -; CHECK-NEXT: [[BLOCK57:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP78:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT58:%.*]] = insertelement <1 x double> undef, double [[TMP78]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP79:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]] -; CHECK-NEXT: [[BLOCK60:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP80:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT61:%.*]] = insertelement <1 x double> undef, double [[TMP80]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT62:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT61]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP81:%.*]] = fmul <1 x double> [[BLOCK60]], [[SPLAT_SPLAT62]] -; CHECK-NEXT: [[TMP82:%.*]] = fadd <1 x double> [[TMP79]], [[TMP81]] -; CHECK-NEXT: [[BLOCK63:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP83:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT64:%.*]] = insertelement <1 x double> undef, double [[TMP83]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT65:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT64]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP84:%.*]] = fmul <1 x double> [[BLOCK63]], [[SPLAT_SPLAT65]] -; CHECK-NEXT: [[TMP85:%.*]] = fadd <1 x double> [[TMP82]], [[TMP84]] -; CHECK-NEXT: [[TMP86:%.*]] = shufflevector <1 x double> [[TMP85]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP87:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP86]], <3 x i32> -; CHECK-NEXT: [[BLOCK66:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP88:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT67:%.*]] = insertelement <1 x double> undef, double [[TMP88]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP89:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]] -; CHECK-NEXT: [[BLOCK69:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP90:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT70:%.*]] = insertelement <1 x double> undef, double [[TMP90]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT71:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT70]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP91:%.*]] = fmul <1 x double> [[BLOCK69]], [[SPLAT_SPLAT71]] -; CHECK-NEXT: [[TMP92:%.*]] = fadd <1 x double> [[TMP89]], [[TMP91]] -; CHECK-NEXT: [[BLOCK72:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP93:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT73:%.*]] = insertelement <1 x double> undef, double [[TMP93]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT74:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT73]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP94:%.*]] = fmul <1 x double> [[BLOCK72]], [[SPLAT_SPLAT74]] -; CHECK-NEXT: [[TMP95:%.*]] = fadd <1 x double> [[TMP92]], [[TMP94]] -; CHECK-NEXT: [[TMP96:%.*]] = shufflevector <1 x double> [[TMP95]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP97:%.*]] = shufflevector <3 x double> [[TMP87]], <3 x double> [[TMP96]], <3 x i32> -; CHECK-NEXT: [[BLOCK75:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP98:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 0 -; CHECK-NEXT: [[SPLAT_SPLATINSERT76:%.*]] = insertelement <1 x double> undef, double [[TMP98]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP99:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]] -; CHECK-NEXT: [[BLOCK78:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP100:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 1 -; CHECK-NEXT: [[SPLAT_SPLATINSERT79:%.*]] = insertelement <1 x double> undef, double [[TMP100]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT80:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT79]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP101:%.*]] = fmul <1 x double> [[BLOCK78]], [[SPLAT_SPLAT80]] -; CHECK-NEXT: [[TMP102:%.*]] = fadd <1 x double> [[TMP99]], [[TMP101]] -; CHECK-NEXT: [[BLOCK81:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <1 x i32> -; CHECK-NEXT: [[TMP103:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 2 -; CHECK-NEXT: [[SPLAT_SPLATINSERT82:%.*]] = insertelement <1 x double> undef, double [[TMP103]], i32 0 -; CHECK-NEXT: [[SPLAT_SPLAT83:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT82]], <1 x double> undef, <1 x i32> zeroinitializer -; CHECK-NEXT: [[TMP104:%.*]] = fmul <1 x double> [[BLOCK81]], [[SPLAT_SPLAT83]] -; CHECK-NEXT: [[TMP105:%.*]] = fadd <1 x double> [[TMP102]], [[TMP104]] - -; Embed result of multiply into flat vector. - -; CHECK-NEXT: [[TMP106:%.*]] = shufflevector <1 x double> [[TMP105]], <1 x double> undef, <3 x i32> -; CHECK-NEXT: [[TMP107:%.*]] = shufflevector <3 x double> [[TMP97]], <3 x double> [[TMP106]], <3 x i32> - -; Load %C. - -; CHECK-NEXT: [[C:%.*]] = load <9 x double>, <9 x double>* [[C_PTR:%.*]] - -; Extract columns from %C. - -; CHECK-NEXT: [[SPLIT84:%.*]] = shufflevector <9 x double> [[C]], <9 x double> undef, <3 x i32> -; CHECK-NEXT: [[SPLIT85:%.*]] = shufflevector <9 x double> [[C]], <9 x double> undef, <3 x i32> -; CHECK-NEXT: [[SPLIT86:%.*]] = shufflevector <9 x double> [[C]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP37:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]] +; CHECK-NEXT: [[BLOCK1:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP38:%.*]] = extractelement <3 x double> [[TMP11]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT2:%.*]] = insertelement <1 x double> undef, double [[TMP38]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT3:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT2]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP39:%.*]] = fmul <1 x double> [[BLOCK1]], [[SPLAT_SPLAT3]] +; CHECK-NEXT: [[TMP40:%.*]] = fadd <1 x double> [[TMP37]], [[TMP39]] +; CHECK-NEXT: [[BLOCK4:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP41:%.*]] = extractelement <3 x double> [[TMP11]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x double> undef, double [[TMP41]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT5]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP42:%.*]] = fmul <1 x double> [[BLOCK4]], [[SPLAT_SPLAT6]] +; CHECK-NEXT: [[TMP43:%.*]] = fadd <1 x double> [[TMP40]], [[TMP42]] +; CHECK-NEXT: [[TMP44:%.*]] = shufflevector <1 x double> [[TMP43]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP45:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP44]], <3 x i32> +; CHECK-NEXT: [[BLOCK7:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP46:%.*]] = extractelement <3 x double> [[TMP11]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT8:%.*]] = insertelement <1 x double> undef, double [[TMP46]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT9:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT8]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP47:%.*]] = fmul <1 x double> [[BLOCK7]], [[SPLAT_SPLAT9]] +; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP48:%.*]] = extractelement <3 x double> [[TMP11]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x double> undef, double [[TMP48]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT11]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP49:%.*]] = fmul <1 x double> [[BLOCK10]], [[SPLAT_SPLAT12]] +; CHECK-NEXT: [[TMP50:%.*]] = fadd <1 x double> [[TMP47]], [[TMP49]] +; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP51:%.*]] = extractelement <3 x double> [[TMP11]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT14:%.*]] = insertelement <1 x double> undef, double [[TMP51]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT14]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP52:%.*]] = fmul <1 x double> [[BLOCK13]], [[SPLAT_SPLAT15]] +; CHECK-NEXT: [[TMP53:%.*]] = fadd <1 x double> [[TMP50]], [[TMP52]] +; CHECK-NEXT: [[TMP54:%.*]] = shufflevector <1 x double> [[TMP53]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP55:%.*]] = shufflevector <3 x double> [[TMP45]], <3 x double> [[TMP54]], <3 x i32> +; CHECK-NEXT: [[BLOCK16:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP56:%.*]] = extractelement <3 x double> [[TMP11]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x double> undef, double [[TMP56]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT17]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP57:%.*]] = fmul <1 x double> [[BLOCK16]], [[SPLAT_SPLAT18]] +; CHECK-NEXT: [[BLOCK19:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP58:%.*]] = extractelement <3 x double> [[TMP11]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT20:%.*]] = insertelement <1 x double> undef, double [[TMP58]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT21:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT20]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP59:%.*]] = fmul <1 x double> [[BLOCK19]], [[SPLAT_SPLAT21]] +; CHECK-NEXT: [[TMP60:%.*]] = fadd <1 x double> [[TMP57]], [[TMP59]] +; CHECK-NEXT: [[BLOCK22:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP61:%.*]] = extractelement <3 x double> [[TMP11]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x double> undef, double [[TMP61]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT23]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP62:%.*]] = fmul <1 x double> [[BLOCK22]], [[SPLAT_SPLAT24]] +; CHECK-NEXT: [[TMP63:%.*]] = fadd <1 x double> [[TMP60]], [[TMP62]] +; CHECK-NEXT: [[TMP64:%.*]] = shufflevector <1 x double> [[TMP63]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP65:%.*]] = shufflevector <3 x double> [[TMP55]], <3 x double> [[TMP64]], <3 x i32> +; CHECK-NEXT: [[BLOCK25:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP66:%.*]] = extractelement <3 x double> [[TMP14]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT26:%.*]] = insertelement <1 x double> undef, double [[TMP66]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT27:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT26]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP67:%.*]] = fmul <1 x double> [[BLOCK25]], [[SPLAT_SPLAT27]] +; CHECK-NEXT: [[BLOCK28:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP68:%.*]] = extractelement <3 x double> [[TMP14]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT29:%.*]] = insertelement <1 x double> undef, double [[TMP68]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT30:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT29]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP69:%.*]] = fmul <1 x double> [[BLOCK28]], [[SPLAT_SPLAT30]] +; CHECK-NEXT: [[TMP70:%.*]] = fadd <1 x double> [[TMP67]], [[TMP69]] +; CHECK-NEXT: [[BLOCK31:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP71:%.*]] = extractelement <3 x double> [[TMP14]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT32:%.*]] = insertelement <1 x double> undef, double [[TMP71]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT33:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT32]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP72:%.*]] = fmul <1 x double> [[BLOCK31]], [[SPLAT_SPLAT33]] +; CHECK-NEXT: [[TMP73:%.*]] = fadd <1 x double> [[TMP70]], [[TMP72]] +; CHECK-NEXT: [[TMP74:%.*]] = shufflevector <1 x double> [[TMP73]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP75:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP74]], <3 x i32> +; CHECK-NEXT: [[BLOCK34:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP76:%.*]] = extractelement <3 x double> [[TMP14]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT35:%.*]] = insertelement <1 x double> undef, double [[TMP76]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT36:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT35]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP77:%.*]] = fmul <1 x double> [[BLOCK34]], [[SPLAT_SPLAT36]] +; CHECK-NEXT: [[BLOCK37:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP78:%.*]] = extractelement <3 x double> [[TMP14]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT38:%.*]] = insertelement <1 x double> undef, double [[TMP78]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT39:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT38]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP79:%.*]] = fmul <1 x double> [[BLOCK37]], [[SPLAT_SPLAT39]] +; CHECK-NEXT: [[TMP80:%.*]] = fadd <1 x double> [[TMP77]], [[TMP79]] +; CHECK-NEXT: [[BLOCK40:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP81:%.*]] = extractelement <3 x double> [[TMP14]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT41:%.*]] = insertelement <1 x double> undef, double [[TMP81]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT42:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT41]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP82:%.*]] = fmul <1 x double> [[BLOCK40]], [[SPLAT_SPLAT42]] +; CHECK-NEXT: [[TMP83:%.*]] = fadd <1 x double> [[TMP80]], [[TMP82]] +; CHECK-NEXT: [[TMP84:%.*]] = shufflevector <1 x double> [[TMP83]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP85:%.*]] = shufflevector <3 x double> [[TMP75]], <3 x double> [[TMP84]], <3 x i32> +; CHECK-NEXT: [[BLOCK43:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP86:%.*]] = extractelement <3 x double> [[TMP14]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT44:%.*]] = insertelement <1 x double> undef, double [[TMP86]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT45:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT44]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP87:%.*]] = fmul <1 x double> [[BLOCK43]], [[SPLAT_SPLAT45]] +; CHECK-NEXT: [[BLOCK46:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP88:%.*]] = extractelement <3 x double> [[TMP14]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT47:%.*]] = insertelement <1 x double> undef, double [[TMP88]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT48:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT47]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP89:%.*]] = fmul <1 x double> [[BLOCK46]], [[SPLAT_SPLAT48]] +; CHECK-NEXT: [[TMP90:%.*]] = fadd <1 x double> [[TMP87]], [[TMP89]] +; CHECK-NEXT: [[BLOCK49:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP91:%.*]] = extractelement <3 x double> [[TMP14]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT50:%.*]] = insertelement <1 x double> undef, double [[TMP91]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT51:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT50]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP92:%.*]] = fmul <1 x double> [[BLOCK49]], [[SPLAT_SPLAT51]] +; CHECK-NEXT: [[TMP93:%.*]] = fadd <1 x double> [[TMP90]], [[TMP92]] +; CHECK-NEXT: [[TMP94:%.*]] = shufflevector <1 x double> [[TMP93]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP95:%.*]] = shufflevector <3 x double> [[TMP85]], <3 x double> [[TMP94]], <3 x i32> +; CHECK-NEXT: [[BLOCK52:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP96:%.*]] = extractelement <3 x double> [[TMP17]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT53:%.*]] = insertelement <1 x double> undef, double [[TMP96]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT54:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT53]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP97:%.*]] = fmul <1 x double> [[BLOCK52]], [[SPLAT_SPLAT54]] +; CHECK-NEXT: [[BLOCK55:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP98:%.*]] = extractelement <3 x double> [[TMP17]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT56:%.*]] = insertelement <1 x double> undef, double [[TMP98]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT57:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT56]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP99:%.*]] = fmul <1 x double> [[BLOCK55]], [[SPLAT_SPLAT57]] +; CHECK-NEXT: [[TMP100:%.*]] = fadd <1 x double> [[TMP97]], [[TMP99]] +; CHECK-NEXT: [[BLOCK58:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP101:%.*]] = extractelement <3 x double> [[TMP17]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT59:%.*]] = insertelement <1 x double> undef, double [[TMP101]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT60:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT59]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP102:%.*]] = fmul <1 x double> [[BLOCK58]], [[SPLAT_SPLAT60]] +; CHECK-NEXT: [[TMP103:%.*]] = fadd <1 x double> [[TMP100]], [[TMP102]] +; CHECK-NEXT: [[TMP104:%.*]] = shufflevector <1 x double> [[TMP103]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP105:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP104]], <3 x i32> +; CHECK-NEXT: [[BLOCK61:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP106:%.*]] = extractelement <3 x double> [[TMP17]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT62:%.*]] = insertelement <1 x double> undef, double [[TMP106]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT63:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT62]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP107:%.*]] = fmul <1 x double> [[BLOCK61]], [[SPLAT_SPLAT63]] +; CHECK-NEXT: [[BLOCK64:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP108:%.*]] = extractelement <3 x double> [[TMP17]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT65:%.*]] = insertelement <1 x double> undef, double [[TMP108]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT66:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT65]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP109:%.*]] = fmul <1 x double> [[BLOCK64]], [[SPLAT_SPLAT66]] +; CHECK-NEXT: [[TMP110:%.*]] = fadd <1 x double> [[TMP107]], [[TMP109]] +; CHECK-NEXT: [[BLOCK67:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP111:%.*]] = extractelement <3 x double> [[TMP17]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT68:%.*]] = insertelement <1 x double> undef, double [[TMP111]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT69:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT68]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP112:%.*]] = fmul <1 x double> [[BLOCK67]], [[SPLAT_SPLAT69]] +; CHECK-NEXT: [[TMP113:%.*]] = fadd <1 x double> [[TMP110]], [[TMP112]] +; CHECK-NEXT: [[TMP114:%.*]] = shufflevector <1 x double> [[TMP113]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP115:%.*]] = shufflevector <3 x double> [[TMP105]], <3 x double> [[TMP114]], <3 x i32> +; CHECK-NEXT: [[BLOCK70:%.*]] = shufflevector <3 x double> [[TMP23]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP116:%.*]] = extractelement <3 x double> [[TMP17]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT71:%.*]] = insertelement <1 x double> undef, double [[TMP116]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT72:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT71]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP117:%.*]] = fmul <1 x double> [[BLOCK70]], [[SPLAT_SPLAT72]] +; CHECK-NEXT: [[BLOCK73:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP118:%.*]] = extractelement <3 x double> [[TMP17]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT74:%.*]] = insertelement <1 x double> undef, double [[TMP118]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT75:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT74]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP119:%.*]] = fmul <1 x double> [[BLOCK73]], [[SPLAT_SPLAT75]] +; CHECK-NEXT: [[TMP120:%.*]] = fadd <1 x double> [[TMP117]], [[TMP119]] +; CHECK-NEXT: [[BLOCK76:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP121:%.*]] = extractelement <3 x double> [[TMP17]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT77:%.*]] = insertelement <1 x double> undef, double [[TMP121]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT78:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT77]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP122:%.*]] = fmul <1 x double> [[BLOCK76]], [[SPLAT_SPLAT78]] +; CHECK-NEXT: [[TMP123:%.*]] = fadd <1 x double> [[TMP120]], [[TMP122]] +; CHECK-NEXT: [[TMP124:%.*]] = shufflevector <1 x double> [[TMP123]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP125:%.*]] = shufflevector <3 x double> [[TMP115]], <3 x double> [[TMP124]], <3 x i32> + +; Load columns of matrix %C. + +; CHECK-NEXT: [[TMP126:%.*]] = bitcast <9 x double>* [[C_PTR:%.*]] to double* +; CHECK-NEXT: [[TMP127:%.*]] = bitcast double* [[TMP126]] to <3 x double>* +; CHECK-NEXT: [[TMP128:%.*]] = load <3 x double>, <3 x double>* [[TMP127]], align 8 +; CHECK-NEXT: [[TMP129:%.*]] = getelementptr double, double* [[TMP126]], i32 3 +; CHECK-NEXT: [[TMP130:%.*]] = bitcast double* [[TMP129]] to <3 x double>* +; CHECK-NEXT: [[TMP131:%.*]] = load <3 x double>, <3 x double>* [[TMP130]], align 8 +; CHECK-NEXT: [[TMP132:%.*]] = getelementptr double, double* [[TMP126]], i32 6 +; CHECK-NEXT: [[TMP133:%.*]] = bitcast double* [[TMP132]] to <3 x double>* +; CHECK-NEXT: [[TMP134:%.*]] = load <3 x double>, <3 x double>* [[TMP133]], align 8 ; Add column vectors. -; CHECK-NEXT: [[TMP108:%.*]] = fadd <3 x double> [[SPLIT84]], [[TMP47]] -; CHECK-NEXT: [[TMP109:%.*]] = fadd <3 x double> [[SPLIT85]], [[TMP77]] -; CHECK-NEXT: [[TMP110:%.*]] = fadd <3 x double> [[SPLIT86]], [[TMP107]] +; CHECK-NEXT: [[TMP135:%.*]] = fadd <3 x double> [[TMP128]], [[TMP65]] +; CHECK-NEXT: [[TMP136:%.*]] = fadd <3 x double> [[TMP131]], [[TMP95]] +; CHECK-NEXT: [[TMP137:%.*]] = fadd <3 x double> [[TMP134]], [[TMP125]] ; Store result columns. -; CHECK-NEXT: [[TMP111:%.*]] = bitcast <9 x double>* [[C_PTR]] to double* -; CHECK-NEXT: [[TMP112:%.*]] = bitcast double* [[TMP111]] to <3 x double>* -; CHECK-NEXT: store <3 x double> [[TMP108]], <3 x double>* [[TMP112]], align 8 -; CHECK-NEXT: [[TMP113:%.*]] = getelementptr double, double* [[TMP111]], i32 3 -; CHECK-NEXT: [[TMP114:%.*]] = bitcast double* [[TMP113]] to <3 x double>* -; CHECK-NEXT: store <3 x double> [[TMP109]], <3 x double>* [[TMP114]], align 8 -; CHECK-NEXT: [[TMP115:%.*]] = getelementptr double, double* [[TMP111]], i32 6 -; CHECK-NEXT: [[TMP116:%.*]] = bitcast double* [[TMP115]] to <3 x double>* -; CHECK-NEXT: store <3 x double> [[TMP110]], <3 x double>* [[TMP116]], align 8 +; CHECK-NEXT: [[TMP138:%.*]] = bitcast <9 x double>* [[C_PTR]] to double* +; CHECK-NEXT: [[TMP139:%.*]] = bitcast double* [[TMP138]] to <3 x double>* +; CHECK-NEXT: store <3 x double> [[TMP135]], <3 x double>* [[TMP139]], align 8 +; CHECK-NEXT: [[TMP140:%.*]] = getelementptr double, double* [[TMP138]], i32 3 +; CHECK-NEXT: [[TMP141:%.*]] = bitcast double* [[TMP140]] to <3 x double>* +; CHECK-NEXT: store <3 x double> [[TMP136]], <3 x double>* [[TMP141]], align 8 +; CHECK-NEXT: [[TMP142:%.*]] = getelementptr double, double* [[TMP138]], i32 6 +; CHECK-NEXT: [[TMP143:%.*]] = bitcast double* [[TMP142]] to <3 x double>* +; CHECK-NEXT: store <3 x double> [[TMP137]], <3 x double>* [[TMP143]], align 8 ; CHECK-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backward.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backward.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backward.ll @@ -0,0 +1,96 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s +; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s + + +define <8 x double> @fadd_transpose(<8 x double> %a, <8 x double> %b) { +; CHECK-LABEL: @fadd_transpose( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLIT5:%.*]] = shufflevector <8 x double> [[B]], <8 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLIT6:%.*]] = shufflevector <8 x double> [[B]], <8 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLIT7:%.*]] = shufflevector <8 x double> [[B]], <8 x double> undef, <2 x i32> +; CHECK-NEXT: [[TMP0:%.*]] = fadd <2 x double> [[SPLIT]], [[SPLIT4]] +; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x double> [[SPLIT1]], [[SPLIT5]] +; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x double> [[SPLIT2]], [[SPLIT6]] +; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[SPLIT3]], [[SPLIT7]] +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP0]], i64 0 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x double> undef, double [[TMP4]], i64 0 +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[TMP1]], i64 0 +; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x double> [[TMP5]], double [[TMP6]], i64 1 +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[TMP2]], i64 0 +; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> [[TMP7]], double [[TMP8]], i64 2 +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP3]], i64 0 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP9]], double [[TMP10]], i64 3 +; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[TMP0]], i64 1 +; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x double> undef, double [[TMP12]], i64 0 +; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[TMP1]], i64 1 +; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP14]], i64 1 +; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[TMP2]], i64 1 +; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP15]], double [[TMP16]], i64 2 +; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x double> [[TMP3]], i64 1 +; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP18]], i64 3 +; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x double> [[TMP11]], <4 x double> [[TMP19]], <8 x i32> +; CHECK-NEXT: ret <8 x double> [[TMP20]] +; +entry: + %add = fadd <8 x double> %a, %b + %c = call <8 x double> @llvm.matrix.transpose(<8 x double> %add, i32 2, i32 4) + ret <8 x double> %c +} + +define <8 x double> @load_fadd_transpose(<8 x double>* %A.Ptr, <8 x double> %b) { +; CHECK-LABEL: @load_fadd_transpose( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x double>* [[A_PTR:%.*]] to double* +; CHECK-NEXT: [[TMP1:%.*]] = bitcast double* [[TMP0]] to <2 x double>* +; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, double* [[TMP0]], i32 2 +; CHECK-NEXT: [[TMP4:%.*]] = bitcast double* [[TMP3]] to <2 x double>* +; CHECK-NEXT: [[TMP5:%.*]] = load <2 x double>, <2 x double>* [[TMP4]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr double, double* [[TMP0]], i32 4 +; CHECK-NEXT: [[TMP7:%.*]] = bitcast double* [[TMP6]] to <2 x double>* +; CHECK-NEXT: [[TMP8:%.*]] = load <2 x double>, <2 x double>* [[TMP7]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr double, double* [[TMP0]], i32 6 +; CHECK-NEXT: [[TMP10:%.*]] = bitcast double* [[TMP9]] to <2 x double>* +; CHECK-NEXT: [[TMP11:%.*]] = load <2 x double>, <2 x double>* [[TMP10]], align 8 +; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <8 x double> [[B]], <8 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <8 x double> [[B]], <8 x double> undef, <2 x i32> +; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <8 x double> [[B]], <8 x double> undef, <2 x i32> +; CHECK-NEXT: [[TMP12:%.*]] = fadd <2 x double> [[TMP2]], [[SPLIT]] +; CHECK-NEXT: [[TMP13:%.*]] = fadd <2 x double> [[TMP5]], [[SPLIT1]] +; CHECK-NEXT: [[TMP14:%.*]] = fadd <2 x double> [[TMP8]], [[SPLIT2]] +; CHECK-NEXT: [[TMP15:%.*]] = fadd <2 x double> [[TMP11]], [[SPLIT3]] +; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[TMP12]], i64 0 +; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> undef, double [[TMP16]], i64 0 +; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x double> [[TMP13]], i64 0 +; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP18]], i64 1 +; CHECK-NEXT: [[TMP20:%.*]] = extractelement <2 x double> [[TMP14]], i64 0 +; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x double> [[TMP19]], double [[TMP20]], i64 2 +; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x double> [[TMP15]], i64 0 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x double> [[TMP21]], double [[TMP22]], i64 3 +; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x double> [[TMP12]], i64 1 +; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x double> undef, double [[TMP24]], i64 0 +; CHECK-NEXT: [[TMP26:%.*]] = extractelement <2 x double> [[TMP13]], i64 1 +; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x double> [[TMP25]], double [[TMP26]], i64 1 +; CHECK-NEXT: [[TMP28:%.*]] = extractelement <2 x double> [[TMP14]], i64 1 +; CHECK-NEXT: [[TMP29:%.*]] = insertelement <4 x double> [[TMP27]], double [[TMP28]], i64 2 +; CHECK-NEXT: [[TMP30:%.*]] = extractelement <2 x double> [[TMP15]], i64 1 +; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x double> [[TMP29]], double [[TMP30]], i64 3 +; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <4 x double> [[TMP23]], <4 x double> [[TMP31]], <8 x i32> +; CHECK-NEXT: ret <8 x double> [[TMP32]] +; +entry: + %a = load <8 x double>, <8 x double>* %A.Ptr + %add = fadd <8 x double> %a, %b + %c = call <8 x double> @llvm.matrix.transpose(<8 x double> %add, i32 2, i32 4) + ret <8 x double> %c +} + + +declare <8 x double> @llvm.matrix.transpose(<8 x double>, i32, i32) diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll @@ -0,0 +1,480 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s +; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s + +define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B, <9 x double> %C) { +; CHECK-LABEL: @unsupported_phi( +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <9 x double> [[A:%.*]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT5:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP0:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <3 x double> undef, double [[TMP0]], i64 0 +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 0 +; CHECK-NEXT: [[TMP3:%.*]] = insertelement <3 x double> [[TMP1]], double [[TMP2]], i64 1 +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 0 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 2 +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 1 +; CHECK-NEXT: [[TMP7:%.*]] = insertelement <3 x double> undef, double [[TMP6]], i64 0 +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 1 +; CHECK-NEXT: [[TMP9:%.*]] = insertelement <3 x double> [[TMP7]], double [[TMP8]], i64 1 +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 1 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 2 +; CHECK-NEXT: [[TMP12:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 2 +; CHECK-NEXT: [[TMP13:%.*]] = insertelement <3 x double> undef, double [[TMP12]], i64 0 +; CHECK-NEXT: [[TMP14:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 2 +; CHECK-NEXT: [[TMP15:%.*]] = insertelement <3 x double> [[TMP13]], double [[TMP14]], i64 1 +; CHECK-NEXT: [[TMP16:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 2 +; CHECK-NEXT: [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 2 +; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> [[TMP11]], <6 x i32> +; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <6 x i32> +; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <6 x double> [[TMP18]], <6 x double> [[TMP19]], <9 x i32> +; CHECK-NEXT: br label [[IF_END:%.*]] +; CHECK: if.else: +; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <9 x double> [[B:%.*]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP21:%.*]] = extractelement <3 x double> [[SPLIT]], i64 0 +; CHECK-NEXT: [[TMP22:%.*]] = insertelement <3 x double> undef, double [[TMP21]], i64 0 +; CHECK-NEXT: [[TMP23:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 0 +; CHECK-NEXT: [[TMP24:%.*]] = insertelement <3 x double> [[TMP22]], double [[TMP23]], i64 1 +; CHECK-NEXT: [[TMP25:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 0 +; CHECK-NEXT: [[TMP26:%.*]] = insertelement <3 x double> [[TMP24]], double [[TMP25]], i64 2 +; CHECK-NEXT: [[TMP27:%.*]] = extractelement <3 x double> [[SPLIT]], i64 1 +; CHECK-NEXT: [[TMP28:%.*]] = insertelement <3 x double> undef, double [[TMP27]], i64 0 +; CHECK-NEXT: [[TMP29:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 1 +; CHECK-NEXT: [[TMP30:%.*]] = insertelement <3 x double> [[TMP28]], double [[TMP29]], i64 1 +; CHECK-NEXT: [[TMP31:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 1 +; CHECK-NEXT: [[TMP32:%.*]] = insertelement <3 x double> [[TMP30]], double [[TMP31]], i64 2 +; CHECK-NEXT: [[TMP33:%.*]] = extractelement <3 x double> [[SPLIT]], i64 2 +; CHECK-NEXT: [[TMP34:%.*]] = insertelement <3 x double> undef, double [[TMP33]], i64 0 +; CHECK-NEXT: [[TMP35:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 2 +; CHECK-NEXT: [[TMP36:%.*]] = insertelement <3 x double> [[TMP34]], double [[TMP35]], i64 1 +; CHECK-NEXT: [[TMP37:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 2 +; CHECK-NEXT: [[TMP38:%.*]] = insertelement <3 x double> [[TMP36]], double [[TMP37]], i64 2 +; CHECK-NEXT: [[TMP39:%.*]] = shufflevector <3 x double> [[TMP26]], <3 x double> [[TMP32]], <6 x i32> +; CHECK-NEXT: [[TMP40:%.*]] = shufflevector <3 x double> [[TMP38]], <3 x double> undef, <6 x i32> +; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <6 x double> [[TMP39]], <6 x double> [[TMP40]], <9 x i32> +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: [[MERGE:%.*]] = phi <9 x double> [ [[TMP20]], [[IF_THEN]] ], [ [[TMP41]], [[IF_ELSE]] ] +; CHECK-NEXT: [[SPLIT6:%.*]] = shufflevector <9 x double> [[C:%.*]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT7:%.*]] = shufflevector <9 x double> [[C]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT8:%.*]] = shufflevector <9 x double> [[C]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT9:%.*]] = shufflevector <9 x double> [[MERGE]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT10:%.*]] = shufflevector <9 x double> [[MERGE]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT11:%.*]] = shufflevector <9 x double> [[MERGE]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP42:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP42]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP43:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]] +; CHECK-NEXT: [[BLOCK12:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP44:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT13:%.*]] = insertelement <1 x double> undef, double [[TMP44]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP45:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]] +; CHECK-NEXT: [[TMP46:%.*]] = fadd <1 x double> [[TMP43]], [[TMP45]] +; CHECK-NEXT: [[BLOCK15:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP47:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT16:%.*]] = insertelement <1 x double> undef, double [[TMP47]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT16]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP48:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]] +; CHECK-NEXT: [[TMP49:%.*]] = fadd <1 x double> [[TMP46]], [[TMP48]] +; CHECK-NEXT: [[TMP50:%.*]] = shufflevector <1 x double> [[TMP49]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP51:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP50]], <3 x i32> +; CHECK-NEXT: [[BLOCK18:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP52:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT19:%.*]] = insertelement <1 x double> undef, double [[TMP52]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT19]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP53:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]] +; CHECK-NEXT: [[BLOCK21:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP54:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT22:%.*]] = insertelement <1 x double> undef, double [[TMP54]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP55:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]] +; CHECK-NEXT: [[TMP56:%.*]] = fadd <1 x double> [[TMP53]], [[TMP55]] +; CHECK-NEXT: [[BLOCK24:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP57:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT25:%.*]] = insertelement <1 x double> undef, double [[TMP57]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT26:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT25]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP58:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]] +; CHECK-NEXT: [[TMP59:%.*]] = fadd <1 x double> [[TMP56]], [[TMP58]] +; CHECK-NEXT: [[TMP60:%.*]] = shufflevector <1 x double> [[TMP59]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP61:%.*]] = shufflevector <3 x double> [[TMP51]], <3 x double> [[TMP60]], <3 x i32> +; CHECK-NEXT: [[BLOCK27:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP62:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT28:%.*]] = insertelement <1 x double> undef, double [[TMP62]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT29:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT28]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP63:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]] +; CHECK-NEXT: [[BLOCK30:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP64:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT31:%.*]] = insertelement <1 x double> undef, double [[TMP64]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP65:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]] +; CHECK-NEXT: [[TMP66:%.*]] = fadd <1 x double> [[TMP63]], [[TMP65]] +; CHECK-NEXT: [[BLOCK33:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP67:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT34:%.*]] = insertelement <1 x double> undef, double [[TMP67]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT35:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT34]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP68:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]] +; CHECK-NEXT: [[TMP69:%.*]] = fadd <1 x double> [[TMP66]], [[TMP68]] +; CHECK-NEXT: [[TMP70:%.*]] = shufflevector <1 x double> [[TMP69]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP71:%.*]] = shufflevector <3 x double> [[TMP61]], <3 x double> [[TMP70]], <3 x i32> +; CHECK-NEXT: [[BLOCK36:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP72:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT37:%.*]] = insertelement <1 x double> undef, double [[TMP72]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT38:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT37]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP73:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]] +; CHECK-NEXT: [[BLOCK39:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP74:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT40:%.*]] = insertelement <1 x double> undef, double [[TMP74]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP75:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]] +; CHECK-NEXT: [[TMP76:%.*]] = fadd <1 x double> [[TMP73]], [[TMP75]] +; CHECK-NEXT: [[BLOCK42:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP77:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT43:%.*]] = insertelement <1 x double> undef, double [[TMP77]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT44:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT43]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP78:%.*]] = fmul <1 x double> [[BLOCK42]], [[SPLAT_SPLAT44]] +; CHECK-NEXT: [[TMP79:%.*]] = fadd <1 x double> [[TMP76]], [[TMP78]] +; CHECK-NEXT: [[TMP80:%.*]] = shufflevector <1 x double> [[TMP79]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP81:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP80]], <3 x i32> +; CHECK-NEXT: [[BLOCK45:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP82:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT46:%.*]] = insertelement <1 x double> undef, double [[TMP82]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT47:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT46]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP83:%.*]] = fmul <1 x double> [[BLOCK45]], [[SPLAT_SPLAT47]] +; CHECK-NEXT: [[BLOCK48:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP84:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT49:%.*]] = insertelement <1 x double> undef, double [[TMP84]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP85:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]] +; CHECK-NEXT: [[TMP86:%.*]] = fadd <1 x double> [[TMP83]], [[TMP85]] +; CHECK-NEXT: [[BLOCK51:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP87:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT52:%.*]] = insertelement <1 x double> undef, double [[TMP87]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT53:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT52]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP88:%.*]] = fmul <1 x double> [[BLOCK51]], [[SPLAT_SPLAT53]] +; CHECK-NEXT: [[TMP89:%.*]] = fadd <1 x double> [[TMP86]], [[TMP88]] +; CHECK-NEXT: [[TMP90:%.*]] = shufflevector <1 x double> [[TMP89]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP91:%.*]] = shufflevector <3 x double> [[TMP81]], <3 x double> [[TMP90]], <3 x i32> +; CHECK-NEXT: [[BLOCK54:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP92:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT55:%.*]] = insertelement <1 x double> undef, double [[TMP92]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT56:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT55]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP93:%.*]] = fmul <1 x double> [[BLOCK54]], [[SPLAT_SPLAT56]] +; CHECK-NEXT: [[BLOCK57:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP94:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT58:%.*]] = insertelement <1 x double> undef, double [[TMP94]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP95:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]] +; CHECK-NEXT: [[TMP96:%.*]] = fadd <1 x double> [[TMP93]], [[TMP95]] +; CHECK-NEXT: [[BLOCK60:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP97:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT61:%.*]] = insertelement <1 x double> undef, double [[TMP97]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT62:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT61]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP98:%.*]] = fmul <1 x double> [[BLOCK60]], [[SPLAT_SPLAT62]] +; CHECK-NEXT: [[TMP99:%.*]] = fadd <1 x double> [[TMP96]], [[TMP98]] +; CHECK-NEXT: [[TMP100:%.*]] = shufflevector <1 x double> [[TMP99]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP101:%.*]] = shufflevector <3 x double> [[TMP91]], <3 x double> [[TMP100]], <3 x i32> +; CHECK-NEXT: [[BLOCK63:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP102:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT64:%.*]] = insertelement <1 x double> undef, double [[TMP102]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT65:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT64]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP103:%.*]] = fmul <1 x double> [[BLOCK63]], [[SPLAT_SPLAT65]] +; CHECK-NEXT: [[BLOCK66:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP104:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT67:%.*]] = insertelement <1 x double> undef, double [[TMP104]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP105:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]] +; CHECK-NEXT: [[TMP106:%.*]] = fadd <1 x double> [[TMP103]], [[TMP105]] +; CHECK-NEXT: [[BLOCK69:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP107:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT70:%.*]] = insertelement <1 x double> undef, double [[TMP107]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT71:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT70]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP108:%.*]] = fmul <1 x double> [[BLOCK69]], [[SPLAT_SPLAT71]] +; CHECK-NEXT: [[TMP109:%.*]] = fadd <1 x double> [[TMP106]], [[TMP108]] +; CHECK-NEXT: [[TMP110:%.*]] = shufflevector <1 x double> [[TMP109]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP111:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP110]], <3 x i32> +; CHECK-NEXT: [[BLOCK72:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP112:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT73:%.*]] = insertelement <1 x double> undef, double [[TMP112]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT74:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT73]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP113:%.*]] = fmul <1 x double> [[BLOCK72]], [[SPLAT_SPLAT74]] +; CHECK-NEXT: [[BLOCK75:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP114:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT76:%.*]] = insertelement <1 x double> undef, double [[TMP114]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP115:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]] +; CHECK-NEXT: [[TMP116:%.*]] = fadd <1 x double> [[TMP113]], [[TMP115]] +; CHECK-NEXT: [[BLOCK78:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP117:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT79:%.*]] = insertelement <1 x double> undef, double [[TMP117]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT80:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT79]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP118:%.*]] = fmul <1 x double> [[BLOCK78]], [[SPLAT_SPLAT80]] +; CHECK-NEXT: [[TMP119:%.*]] = fadd <1 x double> [[TMP116]], [[TMP118]] +; CHECK-NEXT: [[TMP120:%.*]] = shufflevector <1 x double> [[TMP119]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP121:%.*]] = shufflevector <3 x double> [[TMP111]], <3 x double> [[TMP120]], <3 x i32> +; CHECK-NEXT: [[BLOCK81:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP122:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT82:%.*]] = insertelement <1 x double> undef, double [[TMP122]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT83:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT82]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP123:%.*]] = fmul <1 x double> [[BLOCK81]], [[SPLAT_SPLAT83]] +; CHECK-NEXT: [[BLOCK84:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP124:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT85:%.*]] = insertelement <1 x double> undef, double [[TMP124]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT86:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT85]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP125:%.*]] = fmul <1 x double> [[BLOCK84]], [[SPLAT_SPLAT86]] +; CHECK-NEXT: [[TMP126:%.*]] = fadd <1 x double> [[TMP123]], [[TMP125]] +; CHECK-NEXT: [[BLOCK87:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP127:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT88:%.*]] = insertelement <1 x double> undef, double [[TMP127]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT89:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT88]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP128:%.*]] = fmul <1 x double> [[BLOCK87]], [[SPLAT_SPLAT89]] +; CHECK-NEXT: [[TMP129:%.*]] = fadd <1 x double> [[TMP126]], [[TMP128]] +; CHECK-NEXT: [[TMP130:%.*]] = shufflevector <1 x double> [[TMP129]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP131:%.*]] = shufflevector <3 x double> [[TMP121]], <3 x double> [[TMP130]], <3 x i32> +; CHECK-NEXT: [[TMP132:%.*]] = shufflevector <3 x double> [[TMP71]], <3 x double> [[TMP101]], <6 x i32> +; CHECK-NEXT: [[TMP133:%.*]] = shufflevector <3 x double> [[TMP131]], <3 x double> undef, <6 x i32> +; CHECK-NEXT: [[TMP134:%.*]] = shufflevector <6 x double> [[TMP132]], <6 x double> [[TMP133]], <9 x i32> +; CHECK-NEXT: ret <9 x double> [[TMP134]] +; +entry: + br i1 %cond, label %if.then, label %if.else + +if.then: ; preds = %entry + %A.trans = tail call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> %A, i32 3, i32 3) + br label %if.end + +if.else: ; preds = %entry + %B.trans = tail call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> %B, i32 3, i32 3) + br label %if.end + +if.end: ; preds = %if.then, %if.else + %merge = phi <9 x double> [ %A.trans, %if.then], [ %B.trans, %if.else ] + %res = tail call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %C, <9 x double> %merge, i32 3, i32 3, i32 3) + ret <9 x double> %res +} + +; Make sure we use a flattened vector when calling @foo and the use its flat vector result properly. +define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B) { +; CHECK-LABEL: @unsupported_call( +; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <9 x double> [[A:%.*]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = extractelement <3 x double> [[SPLIT]], i64 0 +; CHECK-NEXT: [[TMP2:%.*]] = insertelement <3 x double> undef, double [[TMP1]], i64 0 +; CHECK-NEXT: [[TMP3:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 0 +; CHECK-NEXT: [[TMP4:%.*]] = insertelement <3 x double> [[TMP2]], double [[TMP3]], i64 1 +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 0 +; CHECK-NEXT: [[TMP6:%.*]] = insertelement <3 x double> [[TMP4]], double [[TMP5]], i64 2 +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <3 x double> [[SPLIT]], i64 1 +; CHECK-NEXT: [[TMP8:%.*]] = insertelement <3 x double> undef, double [[TMP7]], i64 0 +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 1 +; CHECK-NEXT: [[TMP10:%.*]] = insertelement <3 x double> [[TMP8]], double [[TMP9]], i64 1 +; CHECK-NEXT: [[TMP11:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 1 +; CHECK-NEXT: [[TMP12:%.*]] = insertelement <3 x double> [[TMP10]], double [[TMP11]], i64 2 +; CHECK-NEXT: [[TMP13:%.*]] = extractelement <3 x double> [[SPLIT]], i64 2 +; CHECK-NEXT: [[TMP14:%.*]] = insertelement <3 x double> undef, double [[TMP13]], i64 0 +; CHECK-NEXT: [[TMP15:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 2 +; CHECK-NEXT: [[TMP16:%.*]] = insertelement <3 x double> [[TMP14]], double [[TMP15]], i64 1 +; CHECK-NEXT: [[TMP17:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 2 +; CHECK-NEXT: [[TMP18:%.*]] = insertelement <3 x double> [[TMP16]], double [[TMP17]], i64 2 +; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <3 x double> [[TMP6]], <3 x double> [[TMP12]], <6 x i32> +; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <3 x double> [[TMP18]], <3 x double> undef, <6 x i32> +; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <6 x double> [[TMP19]], <6 x double> [[TMP20]], <9 x i32> +; CHECK-NEXT: [[A_FOO:%.*]] = call <9 x double> @foo(<9 x double> [[TMP21]]) +; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <9 x double> [[B:%.*]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT5:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT6:%.*]] = shufflevector <9 x double> [[A_FOO]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT7:%.*]] = shufflevector <9 x double> [[A_FOO]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[SPLIT8:%.*]] = shufflevector <9 x double> [[A_FOO]], <9 x double> undef, <3 x i32> +; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP22:%.*]] = extractelement <3 x double> [[SPLIT6]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP22]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP23:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]] +; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <3 x double> [[SPLIT4]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP24:%.*]] = extractelement <3 x double> [[SPLIT6]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> undef, double [[TMP24]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP25:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]] +; CHECK-NEXT: [[TMP26:%.*]] = fadd <1 x double> [[TMP23]], [[TMP25]] +; CHECK-NEXT: [[BLOCK12:%.*]] = shufflevector <3 x double> [[SPLIT5]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP27:%.*]] = extractelement <3 x double> [[SPLIT6]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT13:%.*]] = insertelement <1 x double> undef, double [[TMP27]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP28:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]] +; CHECK-NEXT: [[TMP29:%.*]] = fadd <1 x double> [[TMP26]], [[TMP28]] +; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <1 x double> [[TMP29]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP30]], <3 x i32> +; CHECK-NEXT: [[BLOCK15:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP32:%.*]] = extractelement <3 x double> [[SPLIT6]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT16:%.*]] = insertelement <1 x double> undef, double [[TMP32]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT16]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP33:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]] +; CHECK-NEXT: [[BLOCK18:%.*]] = shufflevector <3 x double> [[SPLIT4]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP34:%.*]] = extractelement <3 x double> [[SPLIT6]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT19:%.*]] = insertelement <1 x double> undef, double [[TMP34]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT19]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP35:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]] +; CHECK-NEXT: [[TMP36:%.*]] = fadd <1 x double> [[TMP33]], [[TMP35]] +; CHECK-NEXT: [[BLOCK21:%.*]] = shufflevector <3 x double> [[SPLIT5]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP37:%.*]] = extractelement <3 x double> [[SPLIT6]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT22:%.*]] = insertelement <1 x double> undef, double [[TMP37]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP38:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]] +; CHECK-NEXT: [[TMP39:%.*]] = fadd <1 x double> [[TMP36]], [[TMP38]] +; CHECK-NEXT: [[TMP40:%.*]] = shufflevector <1 x double> [[TMP39]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <3 x double> [[TMP31]], <3 x double> [[TMP40]], <3 x i32> +; CHECK-NEXT: [[BLOCK24:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP42:%.*]] = extractelement <3 x double> [[SPLIT6]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT25:%.*]] = insertelement <1 x double> undef, double [[TMP42]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT26:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT25]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP43:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]] +; CHECK-NEXT: [[BLOCK27:%.*]] = shufflevector <3 x double> [[SPLIT4]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP44:%.*]] = extractelement <3 x double> [[SPLIT6]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT28:%.*]] = insertelement <1 x double> undef, double [[TMP44]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT29:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT28]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP45:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]] +; CHECK-NEXT: [[TMP46:%.*]] = fadd <1 x double> [[TMP43]], [[TMP45]] +; CHECK-NEXT: [[BLOCK30:%.*]] = shufflevector <3 x double> [[SPLIT5]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP47:%.*]] = extractelement <3 x double> [[SPLIT6]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT31:%.*]] = insertelement <1 x double> undef, double [[TMP47]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP48:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]] +; CHECK-NEXT: [[TMP49:%.*]] = fadd <1 x double> [[TMP46]], [[TMP48]] +; CHECK-NEXT: [[TMP50:%.*]] = shufflevector <1 x double> [[TMP49]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP51:%.*]] = shufflevector <3 x double> [[TMP41]], <3 x double> [[TMP50]], <3 x i32> +; CHECK-NEXT: [[BLOCK33:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP52:%.*]] = extractelement <3 x double> [[SPLIT7]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT34:%.*]] = insertelement <1 x double> undef, double [[TMP52]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT35:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT34]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP53:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]] +; CHECK-NEXT: [[BLOCK36:%.*]] = shufflevector <3 x double> [[SPLIT4]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP54:%.*]] = extractelement <3 x double> [[SPLIT7]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT37:%.*]] = insertelement <1 x double> undef, double [[TMP54]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT38:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT37]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP55:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]] +; CHECK-NEXT: [[TMP56:%.*]] = fadd <1 x double> [[TMP53]], [[TMP55]] +; CHECK-NEXT: [[BLOCK39:%.*]] = shufflevector <3 x double> [[SPLIT5]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP57:%.*]] = extractelement <3 x double> [[SPLIT7]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT40:%.*]] = insertelement <1 x double> undef, double [[TMP57]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP58:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]] +; CHECK-NEXT: [[TMP59:%.*]] = fadd <1 x double> [[TMP56]], [[TMP58]] +; CHECK-NEXT: [[TMP60:%.*]] = shufflevector <1 x double> [[TMP59]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP61:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP60]], <3 x i32> +; CHECK-NEXT: [[BLOCK42:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP62:%.*]] = extractelement <3 x double> [[SPLIT7]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT43:%.*]] = insertelement <1 x double> undef, double [[TMP62]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT44:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT43]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP63:%.*]] = fmul <1 x double> [[BLOCK42]], [[SPLAT_SPLAT44]] +; CHECK-NEXT: [[BLOCK45:%.*]] = shufflevector <3 x double> [[SPLIT4]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP64:%.*]] = extractelement <3 x double> [[SPLIT7]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT46:%.*]] = insertelement <1 x double> undef, double [[TMP64]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT47:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT46]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP65:%.*]] = fmul <1 x double> [[BLOCK45]], [[SPLAT_SPLAT47]] +; CHECK-NEXT: [[TMP66:%.*]] = fadd <1 x double> [[TMP63]], [[TMP65]] +; CHECK-NEXT: [[BLOCK48:%.*]] = shufflevector <3 x double> [[SPLIT5]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP67:%.*]] = extractelement <3 x double> [[SPLIT7]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT49:%.*]] = insertelement <1 x double> undef, double [[TMP67]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP68:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]] +; CHECK-NEXT: [[TMP69:%.*]] = fadd <1 x double> [[TMP66]], [[TMP68]] +; CHECK-NEXT: [[TMP70:%.*]] = shufflevector <1 x double> [[TMP69]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP71:%.*]] = shufflevector <3 x double> [[TMP61]], <3 x double> [[TMP70]], <3 x i32> +; CHECK-NEXT: [[BLOCK51:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP72:%.*]] = extractelement <3 x double> [[SPLIT7]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT52:%.*]] = insertelement <1 x double> undef, double [[TMP72]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT53:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT52]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP73:%.*]] = fmul <1 x double> [[BLOCK51]], [[SPLAT_SPLAT53]] +; CHECK-NEXT: [[BLOCK54:%.*]] = shufflevector <3 x double> [[SPLIT4]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP74:%.*]] = extractelement <3 x double> [[SPLIT7]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT55:%.*]] = insertelement <1 x double> undef, double [[TMP74]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT56:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT55]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP75:%.*]] = fmul <1 x double> [[BLOCK54]], [[SPLAT_SPLAT56]] +; CHECK-NEXT: [[TMP76:%.*]] = fadd <1 x double> [[TMP73]], [[TMP75]] +; CHECK-NEXT: [[BLOCK57:%.*]] = shufflevector <3 x double> [[SPLIT5]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP77:%.*]] = extractelement <3 x double> [[SPLIT7]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT58:%.*]] = insertelement <1 x double> undef, double [[TMP77]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP78:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]] +; CHECK-NEXT: [[TMP79:%.*]] = fadd <1 x double> [[TMP76]], [[TMP78]] +; CHECK-NEXT: [[TMP80:%.*]] = shufflevector <1 x double> [[TMP79]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP81:%.*]] = shufflevector <3 x double> [[TMP71]], <3 x double> [[TMP80]], <3 x i32> +; CHECK-NEXT: [[BLOCK60:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP82:%.*]] = extractelement <3 x double> [[SPLIT8]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT61:%.*]] = insertelement <1 x double> undef, double [[TMP82]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT62:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT61]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP83:%.*]] = fmul <1 x double> [[BLOCK60]], [[SPLAT_SPLAT62]] +; CHECK-NEXT: [[BLOCK63:%.*]] = shufflevector <3 x double> [[SPLIT4]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP84:%.*]] = extractelement <3 x double> [[SPLIT8]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT64:%.*]] = insertelement <1 x double> undef, double [[TMP84]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT65:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT64]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP85:%.*]] = fmul <1 x double> [[BLOCK63]], [[SPLAT_SPLAT65]] +; CHECK-NEXT: [[TMP86:%.*]] = fadd <1 x double> [[TMP83]], [[TMP85]] +; CHECK-NEXT: [[BLOCK66:%.*]] = shufflevector <3 x double> [[SPLIT5]], <3 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP87:%.*]] = extractelement <3 x double> [[SPLIT8]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT67:%.*]] = insertelement <1 x double> undef, double [[TMP87]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP88:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]] +; CHECK-NEXT: [[TMP89:%.*]] = fadd <1 x double> [[TMP86]], [[TMP88]] +; CHECK-NEXT: [[TMP90:%.*]] = shufflevector <1 x double> [[TMP89]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP91:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP90]], <3 x i32> +; CHECK-NEXT: [[BLOCK69:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP92:%.*]] = extractelement <3 x double> [[SPLIT8]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT70:%.*]] = insertelement <1 x double> undef, double [[TMP92]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT71:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT70]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP93:%.*]] = fmul <1 x double> [[BLOCK69]], [[SPLAT_SPLAT71]] +; CHECK-NEXT: [[BLOCK72:%.*]] = shufflevector <3 x double> [[SPLIT4]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP94:%.*]] = extractelement <3 x double> [[SPLIT8]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT73:%.*]] = insertelement <1 x double> undef, double [[TMP94]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT74:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT73]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP95:%.*]] = fmul <1 x double> [[BLOCK72]], [[SPLAT_SPLAT74]] +; CHECK-NEXT: [[TMP96:%.*]] = fadd <1 x double> [[TMP93]], [[TMP95]] +; CHECK-NEXT: [[BLOCK75:%.*]] = shufflevector <3 x double> [[SPLIT5]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP97:%.*]] = extractelement <3 x double> [[SPLIT8]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT76:%.*]] = insertelement <1 x double> undef, double [[TMP97]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP98:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]] +; CHECK-NEXT: [[TMP99:%.*]] = fadd <1 x double> [[TMP96]], [[TMP98]] +; CHECK-NEXT: [[TMP100:%.*]] = shufflevector <1 x double> [[TMP99]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP101:%.*]] = shufflevector <3 x double> [[TMP91]], <3 x double> [[TMP100]], <3 x i32> +; CHECK-NEXT: [[BLOCK78:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP102:%.*]] = extractelement <3 x double> [[SPLIT8]], i64 0 +; CHECK-NEXT: [[SPLAT_SPLATINSERT79:%.*]] = insertelement <1 x double> undef, double [[TMP102]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT80:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT79]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP103:%.*]] = fmul <1 x double> [[BLOCK78]], [[SPLAT_SPLAT80]] +; CHECK-NEXT: [[BLOCK81:%.*]] = shufflevector <3 x double> [[SPLIT4]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP104:%.*]] = extractelement <3 x double> [[SPLIT8]], i64 1 +; CHECK-NEXT: [[SPLAT_SPLATINSERT82:%.*]] = insertelement <1 x double> undef, double [[TMP104]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT83:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT82]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP105:%.*]] = fmul <1 x double> [[BLOCK81]], [[SPLAT_SPLAT83]] +; CHECK-NEXT: [[TMP106:%.*]] = fadd <1 x double> [[TMP103]], [[TMP105]] +; CHECK-NEXT: [[BLOCK84:%.*]] = shufflevector <3 x double> [[SPLIT5]], <3 x double> undef, <1 x i32> +; CHECK-NEXT: [[TMP107:%.*]] = extractelement <3 x double> [[SPLIT8]], i64 2 +; CHECK-NEXT: [[SPLAT_SPLATINSERT85:%.*]] = insertelement <1 x double> undef, double [[TMP107]], i32 0 +; CHECK-NEXT: [[SPLAT_SPLAT86:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT85]], <1 x double> undef, <1 x i32> zeroinitializer +; CHECK-NEXT: [[TMP108:%.*]] = fmul <1 x double> [[BLOCK84]], [[SPLAT_SPLAT86]] +; CHECK-NEXT: [[TMP109:%.*]] = fadd <1 x double> [[TMP106]], [[TMP108]] +; CHECK-NEXT: [[TMP110:%.*]] = shufflevector <1 x double> [[TMP109]], <1 x double> undef, <3 x i32> +; CHECK-NEXT: [[TMP111:%.*]] = shufflevector <3 x double> [[TMP101]], <3 x double> [[TMP110]], <3 x i32> +; CHECK-NEXT: [[TMP112:%.*]] = shufflevector <3 x double> [[TMP51]], <3 x double> [[TMP81]], <6 x i32> +; CHECK-NEXT: [[TMP113:%.*]] = shufflevector <3 x double> [[TMP111]], <3 x double> undef, <6 x i32> +; CHECK-NEXT: [[TMP114:%.*]] = shufflevector <6 x double> [[TMP112]], <6 x double> [[TMP113]], <9 x i32> +; CHECK-NEXT: ret <9 x double> [[TMP114]] +; + %A.trans = tail call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> %A, i32 3, i32 3) + %A.foo = call <9 x double> @foo(<9 x double> %A.trans) + %res = tail call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %B, <9 x double> %A.foo, i32 3, i32 3, i32 3) + ret <9 x double> %res +} + +declare <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double>, <9 x double>, i32 immarg, i32 immarg, i32 immarg) +declare <9 x double> @llvm.matrix.transpose.v9f64(<9 x double>, i32 immarg, i32 immarg) +declare <9 x double> @foo(<9 x double>)