diff --git a/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp b/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp --- a/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp +++ b/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp @@ -73,6 +73,9 @@ Value *Ptr, Value *Stride, Value *Tile); template typename std::enable_if::type createTileDPLoops(BasicBlock *Start, BasicBlock *End, IRBuilderBase &B, @@ -82,6 +85,9 @@ bool lowerTileLoadStore(Instruction *TileLoadStore); template typename std::enable_if::type lowerTileDP(Instruction *TileDP); @@ -223,14 +229,33 @@ template typename std::enable_if::type X86LowerAMXIntrinsics::createTileDPLoops(BasicBlock *Start, BasicBlock *End, IRBuilderBase &B, Value *Row, Value *Col, Value *K, Value *Acc, Value *LHS, Value *RHS) { - std::string IntrinName = - IntrID == Intrinsic::x86_tdpbssd_internal ? "tiledpbssd" : "tdpbf16ps"; + std::string IntrinName; + switch (IntrID) { + case Intrinsic::x86_tdpbssd_internal: + IntrinName = "tiledpbssd"; + break; + case Intrinsic::x86_tdpbsud_internal: + IntrinName = "tiledpbsud"; + break; + case Intrinsic::x86_tdpbusd_internal: + IntrinName = "tiledpbusd"; + break; + case Intrinsic::x86_tdpbuud_internal: + IntrinName = "tiledpbuud"; + break; + case Intrinsic::x86_tdpbf16ps_internal: + IntrinName = "tiledpbf16ps"; + break; + } Loop *RowLoop = nullptr; Loop *ColLoop = nullptr; Loop *InnerLoop = nullptr; @@ -329,7 +354,7 @@ B.CreateAdd(B.CreateMul(CurrentInner, B.getInt16(16)), CurrentCol); Value *NewVecC = nullptr; - if (IntrID == Intrinsic::x86_tdpbssd_internal) { + if (IntrID != Intrinsic::x86_tdpbf16ps_internal) { // tiledpbssd.scalarize.inner.body: // calculate idxa, idxb // %eltc = extractelement <256 x i32> %vec.c.inner.phi, i16 %idxc @@ -351,12 +376,32 @@ Value *SubVecA = B.CreateBitCast(EltA, V4I8Ty); Value *EltB = B.CreateExtractElement(VecB, IdxB); Value *SubVecB = B.CreateBitCast(EltB, V4I8Ty); - Value *SEXTSubVecB = B.CreateSExt(SubVecB, V4I32Ty); - Value *SEXTSubVecA = B.CreateSExt(SubVecA, V4I32Ty); + Value *SEXTSubVecB = nullptr; + Value *SEXTSubVecA = nullptr; + switch (IntrID) { + case Intrinsic::x86_tdpbssd_internal: + SEXTSubVecB = B.CreateSExt(SubVecB, V4I32Ty); + SEXTSubVecA = B.CreateSExt(SubVecA, V4I32Ty); + break; + case Intrinsic::x86_tdpbsud_internal: + SEXTSubVecB = B.CreateZExt(SubVecB, V4I32Ty); + SEXTSubVecA = B.CreateSExt(SubVecA, V4I32Ty); + break; + case Intrinsic::x86_tdpbusd_internal: + SEXTSubVecB = B.CreateSExt(SubVecB, V4I32Ty); + SEXTSubVecA = B.CreateZExt(SubVecA, V4I32Ty); + break; + case Intrinsic::x86_tdpbuud_internal: + SEXTSubVecB = B.CreateZExt(SubVecB, V4I32Ty); + SEXTSubVecA = B.CreateZExt(SubVecA, V4I32Ty); + break; + default: + llvm_unreachable("Invalid intrinsic ID!"); + } Value *SubVecR = B.CreateAddReduce(B.CreateMul(SEXTSubVecA, SEXTSubVecB)); Value *ResElt = B.CreateAdd(EltC, SubVecR); NewVecC = B.CreateInsertElement(VecCPhi, ResElt, IdxC); - } else if (IntrID == Intrinsic::x86_tdpbf16ps_internal) { + } else { // tiledpbf16ps.scalarize.inner.body: // calculate idxa, idxb, idxc // %eltc = extractelement <256 x i32> %vec.c.inner.phi, i16 %idxc @@ -418,6 +463,9 @@ template typename std::enable_if::type X86LowerAMXIntrinsics::lowerTileDP(Instruction *TileDP) { @@ -527,6 +575,9 @@ if (auto *Inst = dyn_cast(&*II++)) { switch (Inst->getIntrinsicID()) { case Intrinsic::x86_tdpbssd_internal: + case Intrinsic::x86_tdpbsud_internal: + case Intrinsic::x86_tdpbusd_internal: + case Intrinsic::x86_tdpbuud_internal: case Intrinsic::x86_tileloadd64_internal: case Intrinsic::x86_tilestored64_internal: case Intrinsic::x86_tilezero_internal: @@ -545,6 +596,15 @@ case Intrinsic::x86_tdpbssd_internal: C = lowerTileDP(Inst) || C; break; + case Intrinsic::x86_tdpbsud_internal: + C = lowerTileDP(Inst) || C; + break; + case Intrinsic::x86_tdpbusd_internal: + C = lowerTileDP(Inst) || C; + break; + case Intrinsic::x86_tdpbuud_internal: + C = lowerTileDP(Inst) || C; + break; case Intrinsic::x86_tdpbf16ps_internal: C = lowerTileDP(Inst) || C; break; diff --git a/llvm/test/CodeGen/X86/AMX/amx-low-intrinsics.ll b/llvm/test/CodeGen/X86/AMX/amx-low-intrinsics.ll --- a/llvm/test/CodeGen/X86/AMX/amx-low-intrinsics.ll +++ b/llvm/test/CodeGen/X86/AMX/amx-low-intrinsics.ll @@ -172,6 +172,231 @@ ret void } +define dso_local void @test_amx_dpbsud(i16 signext %row, i16 signext %col, i16 signext %k, <256 x i32> %c, <256 x i32> %a, <256 x i32> %b, <256 x i32>* %vptr) #0 { +; CHECK-LABEL: @test_amx_dpbsud( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_AMX:%.*]] = bitcast <256 x i32> [[A:%.*]] to x86_amx +; CHECK-NEXT: [[B_AMX:%.*]] = bitcast <256 x i32> [[B:%.*]] to x86_amx +; CHECK-NEXT: [[C_AMX:%.*]] = bitcast <256 x i32> [[C:%.*]] to x86_amx +; CHECK-NEXT: [[TMP0:%.*]] = lshr i16 [[COL:%.*]], 2 +; CHECK-NEXT: [[TMP1:%.*]] = lshr i16 [[K:%.*]], 2 +; CHECK-NEXT: br label [[TILEDPBSUD_SCALARIZE_ROWS_HEADER:%.*]] +; CHECK: tiledpbsud.scalarize.rows.header: +; CHECK-NEXT: [[TILEDPBSUD_SCALARIZE_ROWS_IV:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TILEDPBSUD_SCALARIZE_ROWS_STEP:%.*]], [[TILEDPBSUD_SCALARIZE_ROWS_LATCH:%.*]] ] +; CHECK-NEXT: [[VEC_C_PHI_ROW:%.*]] = phi <256 x i32> [ [[C]], [[ENTRY]] ], [ [[TMP18:%.*]], [[TILEDPBSUD_SCALARIZE_ROWS_LATCH]] ] +; CHECK-NEXT: [[VEC_D_PHI_ROW:%.*]] = phi <256 x i32> [ zeroinitializer, [[ENTRY]] ], [ [[TMP20:%.*]], [[TILEDPBSUD_SCALARIZE_ROWS_LATCH]] ] +; CHECK-NEXT: br label [[TILEDPBSUD_SCALARIZE_ROWS_BODY:%.*]] +; CHECK: tiledpbsud.scalarize.rows.body: +; CHECK-NEXT: br label [[TILEDPBSUD_SCALARIZE_COLS_HEADER:%.*]] +; CHECK: tiledpbsud.scalarize.cols.header: +; CHECK-NEXT: [[TILEDPBSUD_SCALARIZE_COLS_IV:%.*]] = phi i16 [ 0, [[TILEDPBSUD_SCALARIZE_ROWS_BODY]] ], [ [[TILEDPBSUD_SCALARIZE_COLS_STEP:%.*]], [[TILEDPBSUD_SCALARIZE_COLS_LATCH:%.*]] ] +; CHECK-NEXT: [[VEC_C_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_ROW]], [[TILEDPBSUD_SCALARIZE_ROWS_BODY]] ], [ [[TMP18]], [[TILEDPBSUD_SCALARIZE_COLS_LATCH]] ] +; CHECK-NEXT: [[VEC_D_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_D_PHI_ROW]], [[TILEDPBSUD_SCALARIZE_ROWS_BODY]] ], [ [[TMP20]], [[TILEDPBSUD_SCALARIZE_COLS_LATCH]] ] +; CHECK-NEXT: [[TMP2:%.*]] = mul i16 [[TILEDPBSUD_SCALARIZE_ROWS_IV]], 16 +; CHECK-NEXT: [[TMP3:%.*]] = add i16 [[TMP2]], [[TILEDPBSUD_SCALARIZE_COLS_IV]] +; CHECK-NEXT: br label [[TILEDPBSUD_SCALARIZE_COLS_BODY:%.*]] +; CHECK: tiledpbsud.scalarize.cols.body: +; CHECK-NEXT: br label [[TILEDPBSUD_SCALARIZE_INNER_HEADER:%.*]] +; CHECK: tiledpbsud.scalarize.inner.header: +; CHECK-NEXT: [[TILEDPBSUD_SCALARIZE_INNER_IV:%.*]] = phi i16 [ 0, [[TILEDPBSUD_SCALARIZE_COLS_BODY]] ], [ [[TILEDPBSUD_SCALARIZE_INNER_STEP:%.*]], [[TILEDPBSUD_SCALARIZE_INNER_LATCH:%.*]] ] +; CHECK-NEXT: [[VEC_C_INNER_PHI:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_COL]], [[TILEDPBSUD_SCALARIZE_COLS_BODY]] ], [ [[TMP18]], [[TILEDPBSUD_SCALARIZE_INNER_LATCH]] ] +; CHECK-NEXT: br label [[TILEDPBSUD_SCALARIZE_INNER_BODY:%.*]] +; CHECK: tiledpbsud.scalarize.inner.body: +; CHECK-NEXT: [[TMP4:%.*]] = mul i16 [[TILEDPBSUD_SCALARIZE_ROWS_IV]], 16 +; CHECK-NEXT: [[TMP5:%.*]] = add i16 [[TMP4]], [[TILEDPBSUD_SCALARIZE_INNER_IV]] +; CHECK-NEXT: [[TMP6:%.*]] = mul i16 [[TILEDPBSUD_SCALARIZE_INNER_IV]], 16 +; CHECK-NEXT: [[TMP7:%.*]] = add i16 [[TMP6]], [[TILEDPBSUD_SCALARIZE_COLS_IV]] +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <256 x i32> [[VEC_C_INNER_PHI]], i16 [[TMP3]] +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <256 x i32> [[A]], i16 [[TMP5]] +; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP9]] to <4 x i8> +; CHECK-NEXT: [[TMP11:%.*]] = extractelement <256 x i32> [[B]], i16 [[TMP7]] +; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32 [[TMP11]] to <4 x i8> +; CHECK-NEXT: [[TMP13:%.*]] = zext <4 x i8> [[TMP12]] to <4 x i32> +; CHECK-NEXT: [[TMP14:%.*]] = sext <4 x i8> [[TMP10]] to <4 x i32> +; CHECK-NEXT: [[TMP15:%.*]] = mul <4 x i32> [[TMP14]], [[TMP13]] +; CHECK-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]]) +; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP8]], [[TMP16]] +; CHECK-NEXT: [[TMP18]] = insertelement <256 x i32> [[VEC_C_INNER_PHI]], i32 [[TMP17]], i16 [[TMP3]] +; CHECK-NEXT: br label [[TILEDPBSUD_SCALARIZE_INNER_LATCH]] +; CHECK: tiledpbsud.scalarize.inner.latch: +; CHECK-NEXT: [[TILEDPBSUD_SCALARIZE_INNER_STEP]] = add i16 [[TILEDPBSUD_SCALARIZE_INNER_IV]], 1 +; CHECK-NEXT: [[TILEDPBSUD_SCALARIZE_INNER_COND:%.*]] = icmp ne i16 [[TILEDPBSUD_SCALARIZE_INNER_STEP]], [[TMP1]] +; CHECK-NEXT: br i1 [[TILEDPBSUD_SCALARIZE_INNER_COND]], label [[TILEDPBSUD_SCALARIZE_INNER_HEADER]], label [[TILEDPBSUD_SCALARIZE_COLS_LATCH]] +; CHECK: tiledpbsud.scalarize.cols.latch: +; CHECK-NEXT: [[TILEDPBSUD_SCALARIZE_COLS_STEP]] = add i16 [[TILEDPBSUD_SCALARIZE_COLS_IV]], 1 +; CHECK-NEXT: [[TILEDPBSUD_SCALARIZE_COLS_COND:%.*]] = icmp ne i16 [[TILEDPBSUD_SCALARIZE_COLS_STEP]], [[TMP0]] +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <256 x i32> [[TMP18]], i16 [[TMP3]] +; CHECK-NEXT: [[TMP20]] = insertelement <256 x i32> [[VEC_D_PHI_COL]], i32 [[TMP19]], i16 [[TMP3]] +; CHECK-NEXT: br i1 [[TILEDPBSUD_SCALARIZE_COLS_COND]], label [[TILEDPBSUD_SCALARIZE_COLS_HEADER]], label [[TILEDPBSUD_SCALARIZE_ROWS_LATCH]] +; CHECK: tiledpbsud.scalarize.rows.latch: +; CHECK-NEXT: [[TILEDPBSUD_SCALARIZE_ROWS_STEP]] = add i16 [[TILEDPBSUD_SCALARIZE_ROWS_IV]], 1 +; CHECK-NEXT: [[TILEDPBSUD_SCALARIZE_ROWS_COND:%.*]] = icmp ne i16 [[TILEDPBSUD_SCALARIZE_ROWS_STEP]], [[ROW:%.*]] +; CHECK-NEXT: br i1 [[TILEDPBSUD_SCALARIZE_ROWS_COND]], label [[TILEDPBSUD_SCALARIZE_ROWS_HEADER]], label [[CONTINUE:%.*]] +; CHECK: continue: +; CHECK-NEXT: [[TMP21:%.*]] = bitcast <256 x i32> [[TMP20]] to x86_amx +; CHECK-NEXT: store <256 x i32> [[TMP20]], <256 x i32>* [[VPTR:%.*]], align 64 +; CHECK-NEXT: ret void +; +entry: + %a.amx = bitcast <256 x i32> %a to x86_amx + %b.amx = bitcast <256 x i32> %b to x86_amx + %c.amx = bitcast <256 x i32> %c to x86_amx + %acc = call x86_amx @llvm.x86.tdpbsud.internal(i16 %row, i16 %col, i16 %k, x86_amx %c.amx, x86_amx %a.amx, x86_amx %b.amx) + %vec = bitcast x86_amx %acc to <256 x i32> + store <256 x i32> %vec, <256 x i32>* %vptr, align 64 + ret void +} + +define dso_local void @test_amx_dpbusd(i16 signext %row, i16 signext %col, i16 signext %k, <256 x i32> %c, <256 x i32> %a, <256 x i32> %b, <256 x i32>* %vptr) #0 { +; CHECK-LABEL: @test_amx_dpbusd( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_AMX:%.*]] = bitcast <256 x i32> [[A:%.*]] to x86_amx +; CHECK-NEXT: [[B_AMX:%.*]] = bitcast <256 x i32> [[B:%.*]] to x86_amx +; CHECK-NEXT: [[C_AMX:%.*]] = bitcast <256 x i32> [[C:%.*]] to x86_amx +; CHECK-NEXT: [[TMP0:%.*]] = lshr i16 [[COL:%.*]], 2 +; CHECK-NEXT: [[TMP1:%.*]] = lshr i16 [[K:%.*]], 2 +; CHECK-NEXT: br label [[TILEDPBUSD_SCALARIZE_ROWS_HEADER:%.*]] +; CHECK: tiledpbusd.scalarize.rows.header: +; CHECK-NEXT: [[TILEDPBUSD_SCALARIZE_ROWS_IV:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TILEDPBUSD_SCALARIZE_ROWS_STEP:%.*]], [[TILEDPBUSD_SCALARIZE_ROWS_LATCH:%.*]] ] +; CHECK-NEXT: [[VEC_C_PHI_ROW:%.*]] = phi <256 x i32> [ [[C]], [[ENTRY]] ], [ [[TMP18:%.*]], [[TILEDPBUSD_SCALARIZE_ROWS_LATCH]] ] +; CHECK-NEXT: [[VEC_D_PHI_ROW:%.*]] = phi <256 x i32> [ zeroinitializer, [[ENTRY]] ], [ [[TMP20:%.*]], [[TILEDPBUSD_SCALARIZE_ROWS_LATCH]] ] +; CHECK-NEXT: br label [[TILEDPBUSD_SCALARIZE_ROWS_BODY:%.*]] +; CHECK: tiledpbusd.scalarize.rows.body: +; CHECK-NEXT: br label [[TILEDPBUSD_SCALARIZE_COLS_HEADER:%.*]] +; CHECK: tiledpbusd.scalarize.cols.header: +; CHECK-NEXT: [[TILEDPBUSD_SCALARIZE_COLS_IV:%.*]] = phi i16 [ 0, [[TILEDPBUSD_SCALARIZE_ROWS_BODY]] ], [ [[TILEDPBUSD_SCALARIZE_COLS_STEP:%.*]], [[TILEDPBUSD_SCALARIZE_COLS_LATCH:%.*]] ] +; CHECK-NEXT: [[VEC_C_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_ROW]], [[TILEDPBUSD_SCALARIZE_ROWS_BODY]] ], [ [[TMP18]], [[TILEDPBUSD_SCALARIZE_COLS_LATCH]] ] +; CHECK-NEXT: [[VEC_D_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_D_PHI_ROW]], [[TILEDPBUSD_SCALARIZE_ROWS_BODY]] ], [ [[TMP20]], [[TILEDPBUSD_SCALARIZE_COLS_LATCH]] ] +; CHECK-NEXT: [[TMP2:%.*]] = mul i16 [[TILEDPBUSD_SCALARIZE_ROWS_IV]], 16 +; CHECK-NEXT: [[TMP3:%.*]] = add i16 [[TMP2]], [[TILEDPBUSD_SCALARIZE_COLS_IV]] +; CHECK-NEXT: br label [[TILEDPBUSD_SCALARIZE_COLS_BODY:%.*]] +; CHECK: tiledpbusd.scalarize.cols.body: +; CHECK-NEXT: br label [[TILEDPBUSD_SCALARIZE_INNER_HEADER:%.*]] +; CHECK: tiledpbusd.scalarize.inner.header: +; CHECK-NEXT: [[TILEDPBUSD_SCALARIZE_INNER_IV:%.*]] = phi i16 [ 0, [[TILEDPBUSD_SCALARIZE_COLS_BODY]] ], [ [[TILEDPBUSD_SCALARIZE_INNER_STEP:%.*]], [[TILEDPBUSD_SCALARIZE_INNER_LATCH:%.*]] ] +; CHECK-NEXT: [[VEC_C_INNER_PHI:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_COL]], [[TILEDPBUSD_SCALARIZE_COLS_BODY]] ], [ [[TMP18]], [[TILEDPBUSD_SCALARIZE_INNER_LATCH]] ] +; CHECK-NEXT: br label [[TILEDPBUSD_SCALARIZE_INNER_BODY:%.*]] +; CHECK: tiledpbusd.scalarize.inner.body: +; CHECK-NEXT: [[TMP4:%.*]] = mul i16 [[TILEDPBUSD_SCALARIZE_ROWS_IV]], 16 +; CHECK-NEXT: [[TMP5:%.*]] = add i16 [[TMP4]], [[TILEDPBUSD_SCALARIZE_INNER_IV]] +; CHECK-NEXT: [[TMP6:%.*]] = mul i16 [[TILEDPBUSD_SCALARIZE_INNER_IV]], 16 +; CHECK-NEXT: [[TMP7:%.*]] = add i16 [[TMP6]], [[TILEDPBUSD_SCALARIZE_COLS_IV]] +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <256 x i32> [[VEC_C_INNER_PHI]], i16 [[TMP3]] +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <256 x i32> [[A]], i16 [[TMP5]] +; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP9]] to <4 x i8> +; CHECK-NEXT: [[TMP11:%.*]] = extractelement <256 x i32> [[B]], i16 [[TMP7]] +; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32 [[TMP11]] to <4 x i8> +; CHECK-NEXT: [[TMP13:%.*]] = sext <4 x i8> [[TMP12]] to <4 x i32> +; CHECK-NEXT: [[TMP14:%.*]] = zext <4 x i8> [[TMP10]] to <4 x i32> +; CHECK-NEXT: [[TMP15:%.*]] = mul <4 x i32> [[TMP14]], [[TMP13]] +; CHECK-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]]) +; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP8]], [[TMP16]] +; CHECK-NEXT: [[TMP18]] = insertelement <256 x i32> [[VEC_C_INNER_PHI]], i32 [[TMP17]], i16 [[TMP3]] +; CHECK-NEXT: br label [[TILEDPBUSD_SCALARIZE_INNER_LATCH]] +; CHECK: tiledpbusd.scalarize.inner.latch: +; CHECK-NEXT: [[TILEDPBUSD_SCALARIZE_INNER_STEP]] = add i16 [[TILEDPBUSD_SCALARIZE_INNER_IV]], 1 +; CHECK-NEXT: [[TILEDPBUSD_SCALARIZE_INNER_COND:%.*]] = icmp ne i16 [[TILEDPBUSD_SCALARIZE_INNER_STEP]], [[TMP1]] +; CHECK-NEXT: br i1 [[TILEDPBUSD_SCALARIZE_INNER_COND]], label [[TILEDPBUSD_SCALARIZE_INNER_HEADER]], label [[TILEDPBUSD_SCALARIZE_COLS_LATCH]] +; CHECK: tiledpbusd.scalarize.cols.latch: +; CHECK-NEXT: [[TILEDPBUSD_SCALARIZE_COLS_STEP]] = add i16 [[TILEDPBUSD_SCALARIZE_COLS_IV]], 1 +; CHECK-NEXT: [[TILEDPBUSD_SCALARIZE_COLS_COND:%.*]] = icmp ne i16 [[TILEDPBUSD_SCALARIZE_COLS_STEP]], [[TMP0]] +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <256 x i32> [[TMP18]], i16 [[TMP3]] +; CHECK-NEXT: [[TMP20]] = insertelement <256 x i32> [[VEC_D_PHI_COL]], i32 [[TMP19]], i16 [[TMP3]] +; CHECK-NEXT: br i1 [[TILEDPBUSD_SCALARIZE_COLS_COND]], label [[TILEDPBUSD_SCALARIZE_COLS_HEADER]], label [[TILEDPBUSD_SCALARIZE_ROWS_LATCH]] +; CHECK: tiledpbusd.scalarize.rows.latch: +; CHECK-NEXT: [[TILEDPBUSD_SCALARIZE_ROWS_STEP]] = add i16 [[TILEDPBUSD_SCALARIZE_ROWS_IV]], 1 +; CHECK-NEXT: [[TILEDPBUSD_SCALARIZE_ROWS_COND:%.*]] = icmp ne i16 [[TILEDPBUSD_SCALARIZE_ROWS_STEP]], [[ROW:%.*]] +; CHECK-NEXT: br i1 [[TILEDPBUSD_SCALARIZE_ROWS_COND]], label [[TILEDPBUSD_SCALARIZE_ROWS_HEADER]], label [[CONTINUE:%.*]] +; CHECK: continue: +; CHECK-NEXT: [[TMP21:%.*]] = bitcast <256 x i32> [[TMP20]] to x86_amx +; CHECK-NEXT: store <256 x i32> [[TMP20]], <256 x i32>* [[VPTR:%.*]], align 64 +; CHECK-NEXT: ret void +; +entry: + %a.amx = bitcast <256 x i32> %a to x86_amx + %b.amx = bitcast <256 x i32> %b to x86_amx + %c.amx = bitcast <256 x i32> %c to x86_amx + %acc = call x86_amx @llvm.x86.tdpbusd.internal(i16 %row, i16 %col, i16 %k, x86_amx %c.amx, x86_amx %a.amx, x86_amx %b.amx) + %vec = bitcast x86_amx %acc to <256 x i32> + store <256 x i32> %vec, <256 x i32>* %vptr, align 64 + ret void +} + +define dso_local void @test_amx_dpbuud(i16 signext %row, i16 signext %col, i16 signext %k, <256 x i32> %c, <256 x i32> %a, <256 x i32> %b, <256 x i32>* %vptr) #0 { +; CHECK-LABEL: @test_amx_dpbuud( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_AMX:%.*]] = bitcast <256 x i32> [[A:%.*]] to x86_amx +; CHECK-NEXT: [[B_AMX:%.*]] = bitcast <256 x i32> [[B:%.*]] to x86_amx +; CHECK-NEXT: [[C_AMX:%.*]] = bitcast <256 x i32> [[C:%.*]] to x86_amx +; CHECK-NEXT: [[TMP0:%.*]] = lshr i16 [[COL:%.*]], 2 +; CHECK-NEXT: [[TMP1:%.*]] = lshr i16 [[K:%.*]], 2 +; CHECK-NEXT: br label [[TILEDPBUUD_SCALARIZE_ROWS_HEADER:%.*]] +; CHECK: tiledpbuud.scalarize.rows.header: +; CHECK-NEXT: [[TILEDPBUUD_SCALARIZE_ROWS_IV:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TILEDPBUUD_SCALARIZE_ROWS_STEP:%.*]], [[TILEDPBUUD_SCALARIZE_ROWS_LATCH:%.*]] ] +; CHECK-NEXT: [[VEC_C_PHI_ROW:%.*]] = phi <256 x i32> [ [[C]], [[ENTRY]] ], [ [[TMP18:%.*]], [[TILEDPBUUD_SCALARIZE_ROWS_LATCH]] ] +; CHECK-NEXT: [[VEC_D_PHI_ROW:%.*]] = phi <256 x i32> [ zeroinitializer, [[ENTRY]] ], [ [[TMP20:%.*]], [[TILEDPBUUD_SCALARIZE_ROWS_LATCH]] ] +; CHECK-NEXT: br label [[TILEDPBUUD_SCALARIZE_ROWS_BODY:%.*]] +; CHECK: tiledpbuud.scalarize.rows.body: +; CHECK-NEXT: br label [[TILEDPBUUD_SCALARIZE_COLS_HEADER:%.*]] +; CHECK: tiledpbuud.scalarize.cols.header: +; CHECK-NEXT: [[TILEDPBUUD_SCALARIZE_COLS_IV:%.*]] = phi i16 [ 0, [[TILEDPBUUD_SCALARIZE_ROWS_BODY]] ], [ [[TILEDPBUUD_SCALARIZE_COLS_STEP:%.*]], [[TILEDPBUUD_SCALARIZE_COLS_LATCH:%.*]] ] +; CHECK-NEXT: [[VEC_C_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_ROW]], [[TILEDPBUUD_SCALARIZE_ROWS_BODY]] ], [ [[TMP18]], [[TILEDPBUUD_SCALARIZE_COLS_LATCH]] ] +; CHECK-NEXT: [[VEC_D_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_D_PHI_ROW]], [[TILEDPBUUD_SCALARIZE_ROWS_BODY]] ], [ [[TMP20]], [[TILEDPBUUD_SCALARIZE_COLS_LATCH]] ] +; CHECK-NEXT: [[TMP2:%.*]] = mul i16 [[TILEDPBUUD_SCALARIZE_ROWS_IV]], 16 +; CHECK-NEXT: [[TMP3:%.*]] = add i16 [[TMP2]], [[TILEDPBUUD_SCALARIZE_COLS_IV]] +; CHECK-NEXT: br label [[TILEDPBUUD_SCALARIZE_COLS_BODY:%.*]] +; CHECK: tiledpbuud.scalarize.cols.body: +; CHECK-NEXT: br label [[TILEDPBUUD_SCALARIZE_INNER_HEADER:%.*]] +; CHECK: tiledpbuud.scalarize.inner.header: +; CHECK-NEXT: [[TILEDPBUUD_SCALARIZE_INNER_IV:%.*]] = phi i16 [ 0, [[TILEDPBUUD_SCALARIZE_COLS_BODY]] ], [ [[TILEDPBUUD_SCALARIZE_INNER_STEP:%.*]], [[TILEDPBUUD_SCALARIZE_INNER_LATCH:%.*]] ] +; CHECK-NEXT: [[VEC_C_INNER_PHI:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_COL]], [[TILEDPBUUD_SCALARIZE_COLS_BODY]] ], [ [[TMP18]], [[TILEDPBUUD_SCALARIZE_INNER_LATCH]] ] +; CHECK-NEXT: br label [[TILEDPBUUD_SCALARIZE_INNER_BODY:%.*]] +; CHECK: tiledpbuud.scalarize.inner.body: +; CHECK-NEXT: [[TMP4:%.*]] = mul i16 [[TILEDPBUUD_SCALARIZE_ROWS_IV]], 16 +; CHECK-NEXT: [[TMP5:%.*]] = add i16 [[TMP4]], [[TILEDPBUUD_SCALARIZE_INNER_IV]] +; CHECK-NEXT: [[TMP6:%.*]] = mul i16 [[TILEDPBUUD_SCALARIZE_INNER_IV]], 16 +; CHECK-NEXT: [[TMP7:%.*]] = add i16 [[TMP6]], [[TILEDPBUUD_SCALARIZE_COLS_IV]] +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <256 x i32> [[VEC_C_INNER_PHI]], i16 [[TMP3]] +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <256 x i32> [[A]], i16 [[TMP5]] +; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP9]] to <4 x i8> +; CHECK-NEXT: [[TMP11:%.*]] = extractelement <256 x i32> [[B]], i16 [[TMP7]] +; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32 [[TMP11]] to <4 x i8> +; CHECK-NEXT: [[TMP13:%.*]] = zext <4 x i8> [[TMP12]] to <4 x i32> +; CHECK-NEXT: [[TMP14:%.*]] = zext <4 x i8> [[TMP10]] to <4 x i32> +; CHECK-NEXT: [[TMP15:%.*]] = mul <4 x i32> [[TMP14]], [[TMP13]] +; CHECK-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]]) +; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP8]], [[TMP16]] +; CHECK-NEXT: [[TMP18]] = insertelement <256 x i32> [[VEC_C_INNER_PHI]], i32 [[TMP17]], i16 [[TMP3]] +; CHECK-NEXT: br label [[TILEDPBUUD_SCALARIZE_INNER_LATCH]] +; CHECK: tiledpbuud.scalarize.inner.latch: +; CHECK-NEXT: [[TILEDPBUUD_SCALARIZE_INNER_STEP]] = add i16 [[TILEDPBUUD_SCALARIZE_INNER_IV]], 1 +; CHECK-NEXT: [[TILEDPBUUD_SCALARIZE_INNER_COND:%.*]] = icmp ne i16 [[TILEDPBUUD_SCALARIZE_INNER_STEP]], [[TMP1]] +; CHECK-NEXT: br i1 [[TILEDPBUUD_SCALARIZE_INNER_COND]], label [[TILEDPBUUD_SCALARIZE_INNER_HEADER]], label [[TILEDPBUUD_SCALARIZE_COLS_LATCH]] +; CHECK: tiledpbuud.scalarize.cols.latch: +; CHECK-NEXT: [[TILEDPBUUD_SCALARIZE_COLS_STEP]] = add i16 [[TILEDPBUUD_SCALARIZE_COLS_IV]], 1 +; CHECK-NEXT: [[TILEDPBUUD_SCALARIZE_COLS_COND:%.*]] = icmp ne i16 [[TILEDPBUUD_SCALARIZE_COLS_STEP]], [[TMP0]] +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <256 x i32> [[TMP18]], i16 [[TMP3]] +; CHECK-NEXT: [[TMP20]] = insertelement <256 x i32> [[VEC_D_PHI_COL]], i32 [[TMP19]], i16 [[TMP3]] +; CHECK-NEXT: br i1 [[TILEDPBUUD_SCALARIZE_COLS_COND]], label [[TILEDPBUUD_SCALARIZE_COLS_HEADER]], label [[TILEDPBUUD_SCALARIZE_ROWS_LATCH]] +; CHECK: tiledpbuud.scalarize.rows.latch: +; CHECK-NEXT: [[TILEDPBUUD_SCALARIZE_ROWS_STEP]] = add i16 [[TILEDPBUUD_SCALARIZE_ROWS_IV]], 1 +; CHECK-NEXT: [[TILEDPBUUD_SCALARIZE_ROWS_COND:%.*]] = icmp ne i16 [[TILEDPBUUD_SCALARIZE_ROWS_STEP]], [[ROW:%.*]] +; CHECK-NEXT: br i1 [[TILEDPBUUD_SCALARIZE_ROWS_COND]], label [[TILEDPBUUD_SCALARIZE_ROWS_HEADER]], label [[CONTINUE:%.*]] +; CHECK: continue: +; CHECK-NEXT: [[TMP21:%.*]] = bitcast <256 x i32> [[TMP20]] to x86_amx +; CHECK-NEXT: store <256 x i32> [[TMP20]], <256 x i32>* [[VPTR:%.*]], align 64 +; CHECK-NEXT: ret void +; +entry: + %a.amx = bitcast <256 x i32> %a to x86_amx + %b.amx = bitcast <256 x i32> %b to x86_amx + %c.amx = bitcast <256 x i32> %c to x86_amx + %acc = call x86_amx @llvm.x86.tdpbuud.internal(i16 %row, i16 %col, i16 %k, x86_amx %c.amx, x86_amx %a.amx, x86_amx %b.amx) + %vec = bitcast x86_amx %acc to <256 x i32> + store <256 x i32> %vec, <256 x i32>* %vptr, align 64 + ret void +} + define dso_local void @test_amx_dpbf16ps(i16 signext %row, i16 signext %col, i16 signext %k, <256 x i32> %c, <256 x i32> %a, <256 x i32> %b, <256 x i32>* %vptr) #0 { ; CHECK-LABEL: @test_amx_dpbf16ps( ; CHECK-NEXT: entry: @@ -180,32 +405,32 @@ ; CHECK-NEXT: [[C_AMX:%.*]] = bitcast <256 x i32> [[C:%.*]] to x86_amx ; CHECK-NEXT: [[TMP0:%.*]] = lshr i16 [[COL:%.*]], 2 ; CHECK-NEXT: [[TMP1:%.*]] = lshr i16 [[K:%.*]], 2 -; CHECK-NEXT: br label [[TDPBF16PS_SCALARIZE_ROWS_HEADER:%.*]] -; CHECK: tdpbf16ps.scalarize.rows.header: -; CHECK-NEXT: [[TDPBF16PS_SCALARIZE_ROWS_IV:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TDPBF16PS_SCALARIZE_ROWS_STEP:%.*]], [[TDPBF16PS_SCALARIZE_ROWS_LATCH:%.*]] ] -; CHECK-NEXT: [[VEC_C_PHI_ROW:%.*]] = phi <256 x i32> [ [[C]], [[ENTRY]] ], [ [[TMP21:%.*]], [[TDPBF16PS_SCALARIZE_ROWS_LATCH]] ] -; CHECK-NEXT: [[VEC_D_PHI_ROW:%.*]] = phi <256 x i32> [ zeroinitializer, [[ENTRY]] ], [ [[TMP23:%.*]], [[TDPBF16PS_SCALARIZE_ROWS_LATCH]] ] -; CHECK-NEXT: br label [[TDPBF16PS_SCALARIZE_ROWS_BODY:%.*]] -; CHECK: tdpbf16ps.scalarize.rows.body: -; CHECK-NEXT: br label [[TDPBF16PS_SCALARIZE_COLS_HEADER:%.*]] -; CHECK: tdpbf16ps.scalarize.cols.header: -; CHECK-NEXT: [[TDPBF16PS_SCALARIZE_COLS_IV:%.*]] = phi i16 [ 0, [[TDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TDPBF16PS_SCALARIZE_COLS_STEP:%.*]], [[TDPBF16PS_SCALARIZE_COLS_LATCH:%.*]] ] -; CHECK-NEXT: [[VEC_C_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_ROW]], [[TDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TMP21]], [[TDPBF16PS_SCALARIZE_COLS_LATCH]] ] -; CHECK-NEXT: [[VEC_D_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_D_PHI_ROW]], [[TDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TMP23]], [[TDPBF16PS_SCALARIZE_COLS_LATCH]] ] -; CHECK-NEXT: [[TMP2:%.*]] = mul i16 [[TDPBF16PS_SCALARIZE_ROWS_IV]], 16 -; CHECK-NEXT: [[TMP3:%.*]] = add i16 [[TMP2]], [[TDPBF16PS_SCALARIZE_COLS_IV]] -; CHECK-NEXT: br label [[TDPBF16PS_SCALARIZE_COLS_BODY:%.*]] -; CHECK: tdpbf16ps.scalarize.cols.body: -; CHECK-NEXT: br label [[TDPBF16PS_SCALARIZE_INNER_HEADER:%.*]] -; CHECK: tdpbf16ps.scalarize.inner.header: -; CHECK-NEXT: [[TDPBF16PS_SCALARIZE_INNER_IV:%.*]] = phi i16 [ 0, [[TDPBF16PS_SCALARIZE_COLS_BODY]] ], [ [[TDPBF16PS_SCALARIZE_INNER_STEP:%.*]], [[TDPBF16PS_SCALARIZE_INNER_LATCH:%.*]] ] -; CHECK-NEXT: [[VEC_C_INNER_PHI:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_COL]], [[TDPBF16PS_SCALARIZE_COLS_BODY]] ], [ [[TMP21]], [[TDPBF16PS_SCALARIZE_INNER_LATCH]] ] -; CHECK-NEXT: br label [[TDPBF16PS_SCALARIZE_INNER_BODY:%.*]] -; CHECK: tdpbf16ps.scalarize.inner.body: -; CHECK-NEXT: [[TMP4:%.*]] = mul i16 [[TDPBF16PS_SCALARIZE_ROWS_IV]], 16 -; CHECK-NEXT: [[TMP5:%.*]] = add i16 [[TMP4]], [[TDPBF16PS_SCALARIZE_INNER_IV]] -; CHECK-NEXT: [[TMP6:%.*]] = mul i16 [[TDPBF16PS_SCALARIZE_INNER_IV]], 16 -; CHECK-NEXT: [[TMP7:%.*]] = add i16 [[TMP6]], [[TDPBF16PS_SCALARIZE_COLS_IV]] +; CHECK-NEXT: br label [[TILEDPBF16PS_SCALARIZE_ROWS_HEADER:%.*]] +; CHECK: tiledpbf16ps.scalarize.rows.header: +; CHECK-NEXT: [[TILEDPBF16PS_SCALARIZE_ROWS_IV:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TILEDPBF16PS_SCALARIZE_ROWS_STEP:%.*]], [[TILEDPBF16PS_SCALARIZE_ROWS_LATCH:%.*]] ] +; CHECK-NEXT: [[VEC_C_PHI_ROW:%.*]] = phi <256 x i32> [ [[C]], [[ENTRY]] ], [ [[TMP21:%.*]], [[TILEDPBF16PS_SCALARIZE_ROWS_LATCH]] ] +; CHECK-NEXT: [[VEC_D_PHI_ROW:%.*]] = phi <256 x i32> [ zeroinitializer, [[ENTRY]] ], [ [[TMP23:%.*]], [[TILEDPBF16PS_SCALARIZE_ROWS_LATCH]] ] +; CHECK-NEXT: br label [[TILEDPBF16PS_SCALARIZE_ROWS_BODY:%.*]] +; CHECK: tiledpbf16ps.scalarize.rows.body: +; CHECK-NEXT: br label [[TILEDPBF16PS_SCALARIZE_COLS_HEADER:%.*]] +; CHECK: tiledpbf16ps.scalarize.cols.header: +; CHECK-NEXT: [[TILEDPBF16PS_SCALARIZE_COLS_IV:%.*]] = phi i16 [ 0, [[TILEDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TILEDPBF16PS_SCALARIZE_COLS_STEP:%.*]], [[TILEDPBF16PS_SCALARIZE_COLS_LATCH:%.*]] ] +; CHECK-NEXT: [[VEC_C_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_ROW]], [[TILEDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TMP21]], [[TILEDPBF16PS_SCALARIZE_COLS_LATCH]] ] +; CHECK-NEXT: [[VEC_D_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_D_PHI_ROW]], [[TILEDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TMP23]], [[TILEDPBF16PS_SCALARIZE_COLS_LATCH]] ] +; CHECK-NEXT: [[TMP2:%.*]] = mul i16 [[TILEDPBF16PS_SCALARIZE_ROWS_IV]], 16 +; CHECK-NEXT: [[TMP3:%.*]] = add i16 [[TMP2]], [[TILEDPBF16PS_SCALARIZE_COLS_IV]] +; CHECK-NEXT: br label [[TILEDPBF16PS_SCALARIZE_COLS_BODY:%.*]] +; CHECK: tiledpbf16ps.scalarize.cols.body: +; CHECK-NEXT: br label [[TILEDPBF16PS_SCALARIZE_INNER_HEADER:%.*]] +; CHECK: tiledpbf16ps.scalarize.inner.header: +; CHECK-NEXT: [[TILEDPBF16PS_SCALARIZE_INNER_IV:%.*]] = phi i16 [ 0, [[TILEDPBF16PS_SCALARIZE_COLS_BODY]] ], [ [[TILEDPBF16PS_SCALARIZE_INNER_STEP:%.*]], [[TILEDPBF16PS_SCALARIZE_INNER_LATCH:%.*]] ] +; CHECK-NEXT: [[VEC_C_INNER_PHI:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_COL]], [[TILEDPBF16PS_SCALARIZE_COLS_BODY]] ], [ [[TMP21]], [[TILEDPBF16PS_SCALARIZE_INNER_LATCH]] ] +; CHECK-NEXT: br label [[TILEDPBF16PS_SCALARIZE_INNER_BODY:%.*]] +; CHECK: tiledpbf16ps.scalarize.inner.body: +; CHECK-NEXT: [[TMP4:%.*]] = mul i16 [[TILEDPBF16PS_SCALARIZE_ROWS_IV]], 16 +; CHECK-NEXT: [[TMP5:%.*]] = add i16 [[TMP4]], [[TILEDPBF16PS_SCALARIZE_INNER_IV]] +; CHECK-NEXT: [[TMP6:%.*]] = mul i16 [[TILEDPBF16PS_SCALARIZE_INNER_IV]], 16 +; CHECK-NEXT: [[TMP7:%.*]] = add i16 [[TMP6]], [[TILEDPBF16PS_SCALARIZE_COLS_IV]] ; CHECK-NEXT: [[TMP8:%.*]] = extractelement <256 x i32> [[VEC_C_INNER_PHI]], i16 [[TMP3]] ; CHECK-NEXT: [[TMP9:%.*]] = bitcast i32 [[TMP8]] to float ; CHECK-NEXT: [[TMP10:%.*]] = extractelement <256 x i32> [[A]], i16 [[TMP5]] @@ -220,21 +445,21 @@ ; CHECK-NEXT: [[TMP19:%.*]] = call float @llvm.vector.reduce.fadd.v2f32(float [[TMP9]], <2 x float> [[TMP18]]) ; CHECK-NEXT: [[TMP20:%.*]] = bitcast float [[TMP19]] to i32 ; CHECK-NEXT: [[TMP21]] = insertelement <256 x i32> [[VEC_C_INNER_PHI]], i32 [[TMP20]], i16 [[TMP3]] -; CHECK-NEXT: br label [[TDPBF16PS_SCALARIZE_INNER_LATCH]] -; CHECK: tdpbf16ps.scalarize.inner.latch: -; CHECK-NEXT: [[TDPBF16PS_SCALARIZE_INNER_STEP]] = add i16 [[TDPBF16PS_SCALARIZE_INNER_IV]], 1 -; CHECK-NEXT: [[TDPBF16PS_SCALARIZE_INNER_COND:%.*]] = icmp ne i16 [[TDPBF16PS_SCALARIZE_INNER_STEP]], [[TMP1]] -; CHECK-NEXT: br i1 [[TDPBF16PS_SCALARIZE_INNER_COND]], label [[TDPBF16PS_SCALARIZE_INNER_HEADER]], label [[TDPBF16PS_SCALARIZE_COLS_LATCH]] -; CHECK: tdpbf16ps.scalarize.cols.latch: -; CHECK-NEXT: [[TDPBF16PS_SCALARIZE_COLS_STEP]] = add i16 [[TDPBF16PS_SCALARIZE_COLS_IV]], 1 -; CHECK-NEXT: [[TDPBF16PS_SCALARIZE_COLS_COND:%.*]] = icmp ne i16 [[TDPBF16PS_SCALARIZE_COLS_STEP]], [[TMP0]] +; CHECK-NEXT: br label [[TILEDPBF16PS_SCALARIZE_INNER_LATCH]] +; CHECK: tiledpbf16ps.scalarize.inner.latch: +; CHECK-NEXT: [[TILEDPBF16PS_SCALARIZE_INNER_STEP]] = add i16 [[TILEDPBF16PS_SCALARIZE_INNER_IV]], 1 +; CHECK-NEXT: [[TILEDPBF16PS_SCALARIZE_INNER_COND:%.*]] = icmp ne i16 [[TILEDPBF16PS_SCALARIZE_INNER_STEP]], [[TMP1]] +; CHECK-NEXT: br i1 [[TILEDPBF16PS_SCALARIZE_INNER_COND]], label [[TILEDPBF16PS_SCALARIZE_INNER_HEADER]], label [[TILEDPBF16PS_SCALARIZE_COLS_LATCH]] +; CHECK: tiledpbf16ps.scalarize.cols.latch: +; CHECK-NEXT: [[TILEDPBF16PS_SCALARIZE_COLS_STEP]] = add i16 [[TILEDPBF16PS_SCALARIZE_COLS_IV]], 1 +; CHECK-NEXT: [[TILEDPBF16PS_SCALARIZE_COLS_COND:%.*]] = icmp ne i16 [[TILEDPBF16PS_SCALARIZE_COLS_STEP]], [[TMP0]] ; CHECK-NEXT: [[TMP22:%.*]] = extractelement <256 x i32> [[TMP21]], i16 [[TMP3]] ; CHECK-NEXT: [[TMP23]] = insertelement <256 x i32> [[VEC_D_PHI_COL]], i32 [[TMP22]], i16 [[TMP3]] -; CHECK-NEXT: br i1 [[TDPBF16PS_SCALARIZE_COLS_COND]], label [[TDPBF16PS_SCALARIZE_COLS_HEADER]], label [[TDPBF16PS_SCALARIZE_ROWS_LATCH]] -; CHECK: tdpbf16ps.scalarize.rows.latch: -; CHECK-NEXT: [[TDPBF16PS_SCALARIZE_ROWS_STEP]] = add i16 [[TDPBF16PS_SCALARIZE_ROWS_IV]], 1 -; CHECK-NEXT: [[TDPBF16PS_SCALARIZE_ROWS_COND:%.*]] = icmp ne i16 [[TDPBF16PS_SCALARIZE_ROWS_STEP]], [[ROW:%.*]] -; CHECK-NEXT: br i1 [[TDPBF16PS_SCALARIZE_ROWS_COND]], label [[TDPBF16PS_SCALARIZE_ROWS_HEADER]], label [[CONTINUE:%.*]] +; CHECK-NEXT: br i1 [[TILEDPBF16PS_SCALARIZE_COLS_COND]], label [[TILEDPBF16PS_SCALARIZE_COLS_HEADER]], label [[TILEDPBF16PS_SCALARIZE_ROWS_LATCH]] +; CHECK: tiledpbf16ps.scalarize.rows.latch: +; CHECK-NEXT: [[TILEDPBF16PS_SCALARIZE_ROWS_STEP]] = add i16 [[TILEDPBF16PS_SCALARIZE_ROWS_IV]], 1 +; CHECK-NEXT: [[TILEDPBF16PS_SCALARIZE_ROWS_COND:%.*]] = icmp ne i16 [[TILEDPBF16PS_SCALARIZE_ROWS_STEP]], [[ROW:%.*]] +; CHECK-NEXT: br i1 [[TILEDPBF16PS_SCALARIZE_ROWS_COND]], label [[TILEDPBF16PS_SCALARIZE_ROWS_HEADER]], label [[CONTINUE:%.*]] ; CHECK: continue: ; CHECK-NEXT: [[TMP24:%.*]] = bitcast <256 x i32> [[TMP23]] to x86_amx ; CHECK-NEXT: store <256 x i32> [[TMP23]], <256 x i32>* [[VPTR:%.*]], align 64 @@ -310,6 +535,9 @@ declare x86_amx @llvm.x86.tilezero.internal(i16, i16) declare x86_amx @llvm.x86.tileloadd64.internal(i16, i16, i8*, i64) declare x86_amx @llvm.x86.tdpbssd.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx) +declare x86_amx @llvm.x86.tdpbsud.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx) +declare x86_amx @llvm.x86.tdpbusd.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx) +declare x86_amx @llvm.x86.tdpbuud.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx) declare x86_amx @llvm.x86.tdpbf16ps.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx) declare void @llvm.x86.tilestored64.internal(i16, i16, i8*, i64, x86_amx)