[X86] Pass to transform tdpbsud&tdpbusd&tdpbuud intrinsics to scalar operation

[pengfei]

llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp

357

What's this used for? Would it fails when some buildbot only has lower C++ libraries?

llvm/test/CodeGen/X86/AMX/amx-low-intrinsics.ll

409

This change doesn't seem affected by this patch. Do you know which one affects it? Why it still passes without this change?

[yubing]

llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp

357

Ah, you are right. llvm should be built with gcc>=5.1 but if constexpr is not supported in gcc5.1
https://llvm.org/docs/GettingStarted.html
I will delete "constexpr".

llvm/test/CodeGen/X86/AMX/amx-low-intrinsics.ll

409

In fact, the bb's name is not aligned with tiledpbssd. So I changed it to tiledpbf16ps.
You can see it in line255 llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp

[yubing]

Address Pengfei's comments

[pengfei]

LGTM.

[yubing]

It seems there is buildfail, I will revert it and fix bugs.
https://lab.llvm.org/buildbot#builders/77/builds/5053

[krasimir]

PTAL at this build bot breakage:
https://lab.llvm.org/buildbot/#/builders/57/builds/5728

FAILED: lib/Target/X86/CMakeFiles/LLVMX86CodeGen.dir/X86LowerAMXIntrinsics.cpp.o 
/home/buildbots/clang.11.0.0/bin/clang++ --gcc-toolchain=/opt/rh/devtoolset-7/root/usr  -DGTEST_HAS_RTTI=0 -D_DEBUG -D_GNU_SOURCE -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -Ilib/Target/X86 -I/home/buildbots/docker-RHEL-buildbot/SetupBot/worker_env/ppc64le-clang-rhel-test/clang-ppc64le-rhel/llvm/llvm/lib/Target/X86 -Iinclude -I/home/buildbots/docker-RHEL-buildbot/SetupBot/worker_env/ppc64le-clang-rhel-test/clang-ppc64le-rhel/llvm/llvm/include -fPIC -fvisibility-inlines-hidden -Werror -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -pedantic -Wno-long-long -Wc++98-compat-extra-semi -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -fdiagnostics-color -ffunction-sections -fdata-sections -O3  -fPIC    -fno-exceptions -fno-rtti -UNDEBUG -std=c++14 -MD -MT lib/Target/X86/CMakeFiles/LLVMX86CodeGen.dir/X86LowerAMXIntrinsics.cpp.o -MF lib/Target/X86/CMakeFiles/LLVMX86CodeGen.dir/X86LowerAMXIntrinsics.cpp.o.d -o lib/Target/X86/CMakeFiles/LLVMX86CodeGen.dir/X86LowerAMXIntrinsics.cpp.o -c /home/buildbots/docker-RHEL-buildbot/SetupBot/worker_env/ppc64le-clang-rhel-test/clang-ppc64le-rhel/llvm/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp
/home/buildbots/docker-RHEL-buildbot/SetupBot/worker_env/ppc64le-clang-rhel-test/clang-ppc64le-rhel/llvm/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp:381:13: error: no case matching constant switch condition '9824' [-Werror]
    switch (IntrID) {
            ^~~~~~
/home/buildbots/docker-RHEL-buildbot/SetupBot/worker_env/ppc64le-clang-rhel-test/clang-ppc64le-rhel/llvm/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp:485:19: note: in instantiation of function template specialization '(anonymous namespace)::X86LowerAMXIntrinsics::createTileDPLoops<9824>' requested here
  Value *ResVec = createTileDPLoops<IntrID>(Start, End, Builder, M, NDWord,
                  ^
/home/buildbots/docker-RHEL-buildbot/SetupBot/worker_env/ppc64le-clang-rhel-test/clang-ppc64le-rhel/llvm/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp:607:11: note: in instantiation of function template specialization '(anonymous namespace)::X86LowerAMXIntrinsics::lowerTileDP<9824>' requested here
      C = lowerTileDP<Intrinsic::x86_tdpbf16ps_internal>(Inst) || C;
          ^
1 error generated.

[yubing]

Address buildfail in https://lab.llvm.org/buildbot#builders/77/builds/5053

[yubing]

Just a rebase

llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp

BasicBlock *createLoop(BasicBlock *Preheader, BasicBlock *Exit, Value *Bound,
                          Value *Step, StringRef Name, IRBuilderBase &B,
                          Loop *L);
   template <bool IsTileLoad>
   Value *createTileLoadStoreLoops(BasicBlock *Start, BasicBlock *End,
                                   IRBuilderBase &B, Value *Row, Value *Col,
                                   Value *Ptr, Value *Stride, Value *Tile);
   template <Intrinsic::ID IntrID>
   typename std::enable_if<IntrID == Intrinsic::x86_tdpbssd_internal ||
+                              IntrID == Intrinsic::x86_tdpbsud_internal ||
+                              IntrID == Intrinsic::x86_tdpbusd_internal ||
+                              IntrID == Intrinsic::x86_tdpbuud_internal ||
                               IntrID == Intrinsic::x86_tdpbf16ps_internal,
                           Value *>::type
   createTileDPLoops(BasicBlock *Start, BasicBlock *End, IRBuilderBase &B,
                     Value *Row, Value *Col, Value *K, Value *Acc, Value *LHS,
                     Value *RHS);
   template <bool IsTileLoad>
   bool lowerTileLoadStore(Instruction *TileLoadStore);
   template <Intrinsic::ID IntrID>
   typename std::enable_if<IntrID == Intrinsic::x86_tdpbssd_internal ||
+                              IntrID == Intrinsic::x86_tdpbsud_internal ||
+                              IntrID == Intrinsic::x86_tdpbusd_internal ||
+                              IntrID == Intrinsic::x86_tdpbuud_internal ||
                               IntrID == Intrinsic::x86_tdpbf16ps_internal,
                           bool>::type
   lowerTileDP(Instruction *TileDP);
   bool lowerTileZero(Instruction *TileZero);
 };
 
 BasicBlock *X86LowerAMXIntrinsics::createLoop(BasicBlock *Preheader,
                                               BasicBlock *Exit, Value *Bound,
if (IsTileLoad) {
 
     B.CreateStore(Elt, EltPtr);
     return nullptr;
   }
 }
 
 template <Intrinsic::ID IntrID>
 typename std::enable_if<IntrID == Intrinsic::x86_tdpbssd_internal ||
+                            IntrID == Intrinsic::x86_tdpbsud_internal ||
+                            IntrID == Intrinsic::x86_tdpbusd_internal ||
+                            IntrID == Intrinsic::x86_tdpbuud_internal ||
                             IntrID == Intrinsic::x86_tdpbf16ps_internal,
                         Value *>::type
 X86LowerAMXIntrinsics::createTileDPLoops(BasicBlock *Start, BasicBlock *End,
                                          IRBuilderBase &B, Value *Row,
                                          Value *Col, Value *K, Value *Acc,
                                          Value *LHS, Value *RHS) {
-  std::string IntrinName =
-      IntrID == Intrinsic::x86_tdpbssd_internal ? "tiledpbssd" : "tdpbf16ps";
+  std::string IntrinName;
+  switch (IntrID) {
+  case Intrinsic::x86_tdpbssd_internal:
+    IntrinName = "tiledpbssd";
+    break;
+  case Intrinsic::x86_tdpbsud_internal:
+    IntrinName = "tiledpbsud";
+    break;
+  case Intrinsic::x86_tdpbusd_internal:
+    IntrinName = "tiledpbusd";
+    break;
+  case Intrinsic::x86_tdpbuud_internal:
+    IntrinName = "tiledpbuud";
+    break;
+  case Intrinsic::x86_tdpbf16ps_internal:
+    IntrinName = "tiledpbf16ps";
+    break;
+  }
   Loop *RowLoop = nullptr;
   Loop *ColLoop = nullptr;
   Loop *InnerLoop = nullptr;
   if (LI) {
     RowLoop = LI->AllocateLoop();
     ColLoop = LI->AllocateLoop();
     InnerLoop = LI->AllocateLoop();
     ColLoop->addChildLoop(InnerLoop);
X86LowerAMXIntrinsics::createTileDPLoops(BasicBlock *Start, BasicBlock *End,
 
   B.SetInsertPoint(InnerBody->getTerminator());
   Value *IdxA =
       B.CreateAdd(B.CreateMul(CurrentRow, B.getInt16(16)), CurrentInner);
   Value *IdxB =
       B.CreateAdd(B.CreateMul(CurrentInner, B.getInt16(16)), CurrentCol);
   Value *NewVecC = nullptr;
 
-  if (IntrID == Intrinsic::x86_tdpbssd_internal) {
+  if (IntrID != Intrinsic::x86_tdpbf16ps_internal) {
     // tiledpbssd.scalarize.inner.body:
     // calculate idxa, idxb
     // %eltc = extractelement <256 x i32> %vec.c.inner.phi, i16 %idxc
     // %elta = extractelement <256 x i32> %veca, i16 %idxa
     // %eltav4i8 = bitcast i32 %elta to <4 x i8>
     // %eltb = extractelement <256 x i32> %vecb, i16 %idxb
     // %eltbv4i8 = bitcast i32 %eltb to <4 x i8>
     // %eltav4i32 = sext <4 x i8> %eltav4i8 to <4 x i32>
     // %eltbv4i32 = sext <4 x i8> %eltbv4i8 to <4 x i32>
     // %mulab = mul <4 x i32> %eltbv4i32, %eltav4i32
     // %acc = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %131)
     // %neweltc = add i32 %elt, %acc
     // %NewVecC = insertelement <256 x i32> %vec.c.inner.phi, i32 %neweltc,
     // i16 %idxc
     FixedVectorType *V4I8Ty = FixedVectorType::get(B.getInt8Ty(), 4);
     FixedVectorType *V4I32Ty = FixedVectorType::get(B.getInt32Ty(), 4);
     Value *EltC = B.CreateExtractElement(VecCPhi, IdxC);
     Value *EltA = B.CreateExtractElement(VecA, IdxA);
     Value *SubVecA = B.CreateBitCast(EltA, V4I8Ty);
     Value *EltB = B.CreateExtractElement(VecB, IdxB);
     Value *SubVecB = B.CreateBitCast(EltB, V4I8Ty);
-    Value *SEXTSubVecB = B.CreateSExt(SubVecB, V4I32Ty);
-    Value *SEXTSubVecA = B.CreateSExt(SubVecA, V4I32Ty);
+    Value *SEXTSubVecB = nullptr;
+    Value *SEXTSubVecA = nullptr;
+    switch (IntrID) {
+    case Intrinsic::x86_tdpbssd_internal:
+      SEXTSubVecB = B.CreateSExt(SubVecB, V4I32Ty);
+      SEXTSubVecA = B.CreateSExt(SubVecA, V4I32Ty);
+      break;
+    case Intrinsic::x86_tdpbsud_internal:
+      SEXTSubVecB = B.CreateZExt(SubVecB, V4I32Ty);
+      SEXTSubVecA = B.CreateSExt(SubVecA, V4I32Ty);
+      break;
+    case Intrinsic::x86_tdpbusd_internal:
+      SEXTSubVecB = B.CreateSExt(SubVecB, V4I32Ty);
+      SEXTSubVecA = B.CreateZExt(SubVecA, V4I32Ty);
+      break;
+    case Intrinsic::x86_tdpbuud_internal:
+      SEXTSubVecB = B.CreateZExt(SubVecB, V4I32Ty);
+      SEXTSubVecA = B.CreateZExt(SubVecA, V4I32Ty);
+      break;
+    default:
+      llvm_unreachable("Invalid intrinsic ID!");
+    }
     Value *SubVecR = B.CreateAddReduce(B.CreateMul(SEXTSubVecA, SEXTSubVecB));
     Value *ResElt = B.CreateAdd(EltC, SubVecR);
     NewVecC = B.CreateInsertElement(VecCPhi, ResElt, IdxC);
-  } else if (IntrID == Intrinsic::x86_tdpbf16ps_internal) {
+  } else {
     // tiledpbf16ps.scalarize.inner.body:
     // calculate idxa, idxb, idxc
     // %eltc = extractelement <256 x i32> %vec.c.inner.phi, i16 %idxc
     // %eltcf32 = bitcast i32 %eltc to float
     // %elta = extractelement <256 x i32> %veca, i16 %idxa
     // %eltav2i16 = bitcast i32 %elta to <2 x i16>
     // %eltb = extractelement <256 x i32> %vecb, i16 %idxb
     // %eltbv2i16 = bitcast i32 %eltb to <2 x i16>
X86LowerAMXIntrinsics::createTileDPLoops(BasicBlock *Start, BasicBlock *End,
   VecDPhiRowLoop->addIncoming(NewVecD, RowLatch);
   VecDPhiColLoop->addIncoming(NewVecD, ColLoopLatch);
 
   return NewVecD;
 }
 
 template <Intrinsic::ID IntrID>
 typename std::enable_if<IntrID == Intrinsic::x86_tdpbssd_internal ||
+                            IntrID == Intrinsic::x86_tdpbsud_internal ||
+                            IntrID == Intrinsic::x86_tdpbusd_internal ||
+                            IntrID == Intrinsic::x86_tdpbuud_internal ||
                             IntrID == Intrinsic::x86_tdpbf16ps_internal,
                         bool>::type
 X86LowerAMXIntrinsics::lowerTileDP(Instruction *TileDP) {
   Value *M, *N, *K, *C, *A, *B;
   match(TileDP, m_Intrinsic<IntrID>(m_Value(M), m_Value(N), m_Value(K),
                                     m_Value(C), m_Value(A), m_Value(B)));
   Instruction *InsertI = TileDP;
   IRBuilder<> PreBuilder(TileDP);

 bool X86LowerAMXIntrinsics::visit() {
   bool C = false;
   SmallVector<IntrinsicInst *, 8> WorkList;
   for (BasicBlock *BB : depth_first(&Func)) {
     for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) {
       if (auto *Inst = dyn_cast<IntrinsicInst>(&*II++)) {
         switch (Inst->getIntrinsicID()) {
         case Intrinsic::x86_tdpbssd_internal:
+        case Intrinsic::x86_tdpbsud_internal:
+        case Intrinsic::x86_tdpbusd_internal:
+        case Intrinsic::x86_tdpbuud_internal:
         case Intrinsic::x86_tileloadd64_internal:
         case Intrinsic::x86_tilestored64_internal:
         case Intrinsic::x86_tilezero_internal:
         case Intrinsic::x86_tdpbf16ps_internal:
           WorkList.push_back(Inst);
           break;
         default:
           break;
         }
       }
     }
   }
 
   for (auto *Inst : WorkList) {
     switch (Inst->getIntrinsicID()) {
     case Intrinsic::x86_tdpbssd_internal:
       C = lowerTileDP<Intrinsic::x86_tdpbssd_internal>(Inst) || C;
       break;
+    case Intrinsic::x86_tdpbsud_internal:
+      C = lowerTileDP<Intrinsic::x86_tdpbsud_internal>(Inst) || C;
+      break;
+    case Intrinsic::x86_tdpbusd_internal:
+      C = lowerTileDP<Intrinsic::x86_tdpbusd_internal>(Inst) || C;
+      break;
+    case Intrinsic::x86_tdpbuud_internal:
+      C = lowerTileDP<Intrinsic::x86_tdpbuud_internal>(Inst) || C;
+      break;
     case Intrinsic::x86_tdpbf16ps_internal:
       C = lowerTileDP<Intrinsic::x86_tdpbf16ps_internal>(Inst) || C;
       break;
     case Intrinsic::x86_tileloadd64_internal:
       C = lowerTileLoadStore<true>(Inst) || C;
       break;
     case Intrinsic::x86_tilestored64_internal:
       C = lowerTileLoadStore<false>(Inst) || C;

llvm/test/CodeGen/X86/AMX/amx-low-intrinsics.ll

entry:
   %b.amx = bitcast <256 x i32> %b to x86_amx
   %c.amx = bitcast <256 x i32> %c to x86_amx
   %acc = call x86_amx @llvm.x86.tdpbssd.internal(i16 %row, i16 %col, i16 %k, x86_amx %c.amx, x86_amx %a.amx, x86_amx %b.amx)
   %vec = bitcast x86_amx %acc to <256 x i32>
   store <256 x i32> %vec, <256 x i32>* %vptr, align 64
   ret void
 }
 
+define dso_local void @test_amx_dpbsud(i16 signext %row, i16 signext %col, i16 signext %k, <256 x i32> %c, <256 x i32> %a, <256 x i32> %b, <256 x i32>* %vptr) #0 {
+; CHECK-LABEL: @test_amx_dpbsud(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_AMX:%.*]] = bitcast <256 x i32> [[A:%.*]] to x86_amx
+; CHECK-NEXT:    [[B_AMX:%.*]] = bitcast <256 x i32> [[B:%.*]] to x86_amx
+; CHECK-NEXT:    [[C_AMX:%.*]] = bitcast <256 x i32> [[C:%.*]] to x86_amx
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i16 [[COL:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i16 [[K:%.*]], 2
+; CHECK-NEXT:    br label [[TILEDPBSUD_SCALARIZE_ROWS_HEADER:%.*]]
+; CHECK:       tiledpbsud.scalarize.rows.header:
+; CHECK-NEXT:    [[TILEDPBSUD_SCALARIZE_ROWS_IV:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TILEDPBSUD_SCALARIZE_ROWS_STEP:%.*]], [[TILEDPBSUD_SCALARIZE_ROWS_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_PHI_ROW:%.*]] = phi <256 x i32> [ [[C]], [[ENTRY]] ], [ [[TMP18:%.*]], [[TILEDPBSUD_SCALARIZE_ROWS_LATCH]] ]
+; CHECK-NEXT:    [[VEC_D_PHI_ROW:%.*]] = phi <256 x i32> [ zeroinitializer, [[ENTRY]] ], [ [[TMP20:%.*]], [[TILEDPBSUD_SCALARIZE_ROWS_LATCH]] ]
+; CHECK-NEXT:    br label [[TILEDPBSUD_SCALARIZE_ROWS_BODY:%.*]]
+; CHECK:       tiledpbsud.scalarize.rows.body:
+; CHECK-NEXT:    br label [[TILEDPBSUD_SCALARIZE_COLS_HEADER:%.*]]
+; CHECK:       tiledpbsud.scalarize.cols.header:
+; CHECK-NEXT:    [[TILEDPBSUD_SCALARIZE_COLS_IV:%.*]] = phi i16 [ 0, [[TILEDPBSUD_SCALARIZE_ROWS_BODY]] ], [ [[TILEDPBSUD_SCALARIZE_COLS_STEP:%.*]], [[TILEDPBSUD_SCALARIZE_COLS_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_ROW]], [[TILEDPBSUD_SCALARIZE_ROWS_BODY]] ], [ [[TMP18]], [[TILEDPBSUD_SCALARIZE_COLS_LATCH]] ]
+; CHECK-NEXT:    [[VEC_D_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_D_PHI_ROW]], [[TILEDPBSUD_SCALARIZE_ROWS_BODY]] ], [ [[TMP20]], [[TILEDPBSUD_SCALARIZE_COLS_LATCH]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i16 [[TILEDPBSUD_SCALARIZE_ROWS_IV]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = add i16 [[TMP2]], [[TILEDPBSUD_SCALARIZE_COLS_IV]]
+; CHECK-NEXT:    br label [[TILEDPBSUD_SCALARIZE_COLS_BODY:%.*]]
+; CHECK:       tiledpbsud.scalarize.cols.body:
+; CHECK-NEXT:    br label [[TILEDPBSUD_SCALARIZE_INNER_HEADER:%.*]]
+; CHECK:       tiledpbsud.scalarize.inner.header:
+; CHECK-NEXT:    [[TILEDPBSUD_SCALARIZE_INNER_IV:%.*]] = phi i16 [ 0, [[TILEDPBSUD_SCALARIZE_COLS_BODY]] ], [ [[TILEDPBSUD_SCALARIZE_INNER_STEP:%.*]], [[TILEDPBSUD_SCALARIZE_INNER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_INNER_PHI:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_COL]], [[TILEDPBSUD_SCALARIZE_COLS_BODY]] ], [ [[TMP18]], [[TILEDPBSUD_SCALARIZE_INNER_LATCH]] ]
+; CHECK-NEXT:    br label [[TILEDPBSUD_SCALARIZE_INNER_BODY:%.*]]
+; CHECK:       tiledpbsud.scalarize.inner.body:
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i16 [[TILEDPBSUD_SCALARIZE_ROWS_IV]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = add i16 [[TMP4]], [[TILEDPBSUD_SCALARIZE_INNER_IV]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i16 [[TILEDPBSUD_SCALARIZE_INNER_IV]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = add i16 [[TMP6]], [[TILEDPBSUD_SCALARIZE_COLS_IV]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <256 x i32> [[VEC_C_INNER_PHI]], i16 [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <256 x i32> [[A]], i16 [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP9]] to <4 x i8>
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <256 x i32> [[B]], i16 [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[TMP11]] to <4 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = zext <4 x i8> [[TMP12]] to <4 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = sext <4 x i8> [[TMP10]] to <4 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = mul <4 x i32> [[TMP14]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]])
+; CHECK-NEXT:    [[TMP17:%.*]] = add i32 [[TMP8]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18]] = insertelement <256 x i32> [[VEC_C_INNER_PHI]], i32 [[TMP17]], i16 [[TMP3]]
+; CHECK-NEXT:    br label [[TILEDPBSUD_SCALARIZE_INNER_LATCH]]
+; CHECK:       tiledpbsud.scalarize.inner.latch:
+; CHECK-NEXT:    [[TILEDPBSUD_SCALARIZE_INNER_STEP]] = add i16 [[TILEDPBSUD_SCALARIZE_INNER_IV]], 1
+; CHECK-NEXT:    [[TILEDPBSUD_SCALARIZE_INNER_COND:%.*]] = icmp ne i16 [[TILEDPBSUD_SCALARIZE_INNER_STEP]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[TILEDPBSUD_SCALARIZE_INNER_COND]], label [[TILEDPBSUD_SCALARIZE_INNER_HEADER]], label [[TILEDPBSUD_SCALARIZE_COLS_LATCH]]
+; CHECK:       tiledpbsud.scalarize.cols.latch:
+; CHECK-NEXT:    [[TILEDPBSUD_SCALARIZE_COLS_STEP]] = add i16 [[TILEDPBSUD_SCALARIZE_COLS_IV]], 1
+; CHECK-NEXT:    [[TILEDPBSUD_SCALARIZE_COLS_COND:%.*]] = icmp ne i16 [[TILEDPBSUD_SCALARIZE_COLS_STEP]], [[TMP0]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <256 x i32> [[TMP18]], i16 [[TMP3]]
+; CHECK-NEXT:    [[TMP20]] = insertelement <256 x i32> [[VEC_D_PHI_COL]], i32 [[TMP19]], i16 [[TMP3]]
+; CHECK-NEXT:    br i1 [[TILEDPBSUD_SCALARIZE_COLS_COND]], label [[TILEDPBSUD_SCALARIZE_COLS_HEADER]], label [[TILEDPBSUD_SCALARIZE_ROWS_LATCH]]
+; CHECK:       tiledpbsud.scalarize.rows.latch:
+; CHECK-NEXT:    [[TILEDPBSUD_SCALARIZE_ROWS_STEP]] = add i16 [[TILEDPBSUD_SCALARIZE_ROWS_IV]], 1
+; CHECK-NEXT:    [[TILEDPBSUD_SCALARIZE_ROWS_COND:%.*]] = icmp ne i16 [[TILEDPBSUD_SCALARIZE_ROWS_STEP]], [[ROW:%.*]]
+; CHECK-NEXT:    br i1 [[TILEDPBSUD_SCALARIZE_ROWS_COND]], label [[TILEDPBSUD_SCALARIZE_ROWS_HEADER]], label [[CONTINUE:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <256 x i32> [[TMP20]] to x86_amx
+; CHECK-NEXT:    store <256 x i32> [[TMP20]], <256 x i32>* [[VPTR:%.*]], align 64
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a.amx = bitcast <256 x i32> %a to x86_amx
+  %b.amx = bitcast <256 x i32> %b to x86_amx
+  %c.amx = bitcast <256 x i32> %c to x86_amx
+  %acc = call x86_amx @llvm.x86.tdpbsud.internal(i16 %row, i16 %col, i16 %k, x86_amx %c.amx, x86_amx %a.amx, x86_amx %b.amx)
+  %vec = bitcast x86_amx %acc to <256 x i32>
+  store <256 x i32> %vec, <256 x i32>* %vptr, align 64
+  ret void
+}
+
+define dso_local void @test_amx_dpbusd(i16 signext %row, i16 signext %col, i16 signext %k, <256 x i32> %c, <256 x i32> %a, <256 x i32> %b, <256 x i32>* %vptr) #0 {
+; CHECK-LABEL: @test_amx_dpbusd(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_AMX:%.*]] = bitcast <256 x i32> [[A:%.*]] to x86_amx
+; CHECK-NEXT:    [[B_AMX:%.*]] = bitcast <256 x i32> [[B:%.*]] to x86_amx
+; CHECK-NEXT:    [[C_AMX:%.*]] = bitcast <256 x i32> [[C:%.*]] to x86_amx
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i16 [[COL:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i16 [[K:%.*]], 2
+; CHECK-NEXT:    br label [[TILEDPBUSD_SCALARIZE_ROWS_HEADER:%.*]]
+; CHECK:       tiledpbusd.scalarize.rows.header:
+; CHECK-NEXT:    [[TILEDPBUSD_SCALARIZE_ROWS_IV:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TILEDPBUSD_SCALARIZE_ROWS_STEP:%.*]], [[TILEDPBUSD_SCALARIZE_ROWS_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_PHI_ROW:%.*]] = phi <256 x i32> [ [[C]], [[ENTRY]] ], [ [[TMP18:%.*]], [[TILEDPBUSD_SCALARIZE_ROWS_LATCH]] ]
+; CHECK-NEXT:    [[VEC_D_PHI_ROW:%.*]] = phi <256 x i32> [ zeroinitializer, [[ENTRY]] ], [ [[TMP20:%.*]], [[TILEDPBUSD_SCALARIZE_ROWS_LATCH]] ]
+; CHECK-NEXT:    br label [[TILEDPBUSD_SCALARIZE_ROWS_BODY:%.*]]
+; CHECK:       tiledpbusd.scalarize.rows.body:
+; CHECK-NEXT:    br label [[TILEDPBUSD_SCALARIZE_COLS_HEADER:%.*]]
+; CHECK:       tiledpbusd.scalarize.cols.header:
+; CHECK-NEXT:    [[TILEDPBUSD_SCALARIZE_COLS_IV:%.*]] = phi i16 [ 0, [[TILEDPBUSD_SCALARIZE_ROWS_BODY]] ], [ [[TILEDPBUSD_SCALARIZE_COLS_STEP:%.*]], [[TILEDPBUSD_SCALARIZE_COLS_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_ROW]], [[TILEDPBUSD_SCALARIZE_ROWS_BODY]] ], [ [[TMP18]], [[TILEDPBUSD_SCALARIZE_COLS_LATCH]] ]
+; CHECK-NEXT:    [[VEC_D_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_D_PHI_ROW]], [[TILEDPBUSD_SCALARIZE_ROWS_BODY]] ], [ [[TMP20]], [[TILEDPBUSD_SCALARIZE_COLS_LATCH]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i16 [[TILEDPBUSD_SCALARIZE_ROWS_IV]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = add i16 [[TMP2]], [[TILEDPBUSD_SCALARIZE_COLS_IV]]
+; CHECK-NEXT:    br label [[TILEDPBUSD_SCALARIZE_COLS_BODY:%.*]]
+; CHECK:       tiledpbusd.scalarize.cols.body:
+; CHECK-NEXT:    br label [[TILEDPBUSD_SCALARIZE_INNER_HEADER:%.*]]
+; CHECK:       tiledpbusd.scalarize.inner.header:
+; CHECK-NEXT:    [[TILEDPBUSD_SCALARIZE_INNER_IV:%.*]] = phi i16 [ 0, [[TILEDPBUSD_SCALARIZE_COLS_BODY]] ], [ [[TILEDPBUSD_SCALARIZE_INNER_STEP:%.*]], [[TILEDPBUSD_SCALARIZE_INNER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_INNER_PHI:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_COL]], [[TILEDPBUSD_SCALARIZE_COLS_BODY]] ], [ [[TMP18]], [[TILEDPBUSD_SCALARIZE_INNER_LATCH]] ]
+; CHECK-NEXT:    br label [[TILEDPBUSD_SCALARIZE_INNER_BODY:%.*]]
+; CHECK:       tiledpbusd.scalarize.inner.body:
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i16 [[TILEDPBUSD_SCALARIZE_ROWS_IV]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = add i16 [[TMP4]], [[TILEDPBUSD_SCALARIZE_INNER_IV]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i16 [[TILEDPBUSD_SCALARIZE_INNER_IV]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = add i16 [[TMP6]], [[TILEDPBUSD_SCALARIZE_COLS_IV]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <256 x i32> [[VEC_C_INNER_PHI]], i16 [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <256 x i32> [[A]], i16 [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP9]] to <4 x i8>
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <256 x i32> [[B]], i16 [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[TMP11]] to <4 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = sext <4 x i8> [[TMP12]] to <4 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = zext <4 x i8> [[TMP10]] to <4 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = mul <4 x i32> [[TMP14]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]])
+; CHECK-NEXT:    [[TMP17:%.*]] = add i32 [[TMP8]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18]] = insertelement <256 x i32> [[VEC_C_INNER_PHI]], i32 [[TMP17]], i16 [[TMP3]]
+; CHECK-NEXT:    br label [[TILEDPBUSD_SCALARIZE_INNER_LATCH]]
+; CHECK:       tiledpbusd.scalarize.inner.latch:
+; CHECK-NEXT:    [[TILEDPBUSD_SCALARIZE_INNER_STEP]] = add i16 [[TILEDPBUSD_SCALARIZE_INNER_IV]], 1
+; CHECK-NEXT:    [[TILEDPBUSD_SCALARIZE_INNER_COND:%.*]] = icmp ne i16 [[TILEDPBUSD_SCALARIZE_INNER_STEP]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[TILEDPBUSD_SCALARIZE_INNER_COND]], label [[TILEDPBUSD_SCALARIZE_INNER_HEADER]], label [[TILEDPBUSD_SCALARIZE_COLS_LATCH]]
+; CHECK:       tiledpbusd.scalarize.cols.latch:
+; CHECK-NEXT:    [[TILEDPBUSD_SCALARIZE_COLS_STEP]] = add i16 [[TILEDPBUSD_SCALARIZE_COLS_IV]], 1
+; CHECK-NEXT:    [[TILEDPBUSD_SCALARIZE_COLS_COND:%.*]] = icmp ne i16 [[TILEDPBUSD_SCALARIZE_COLS_STEP]], [[TMP0]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <256 x i32> [[TMP18]], i16 [[TMP3]]
+; CHECK-NEXT:    [[TMP20]] = insertelement <256 x i32> [[VEC_D_PHI_COL]], i32 [[TMP19]], i16 [[TMP3]]
+; CHECK-NEXT:    br i1 [[TILEDPBUSD_SCALARIZE_COLS_COND]], label [[TILEDPBUSD_SCALARIZE_COLS_HEADER]], label [[TILEDPBUSD_SCALARIZE_ROWS_LATCH]]
+; CHECK:       tiledpbusd.scalarize.rows.latch:
+; CHECK-NEXT:    [[TILEDPBUSD_SCALARIZE_ROWS_STEP]] = add i16 [[TILEDPBUSD_SCALARIZE_ROWS_IV]], 1
+; CHECK-NEXT:    [[TILEDPBUSD_SCALARIZE_ROWS_COND:%.*]] = icmp ne i16 [[TILEDPBUSD_SCALARIZE_ROWS_STEP]], [[ROW:%.*]]
+; CHECK-NEXT:    br i1 [[TILEDPBUSD_SCALARIZE_ROWS_COND]], label [[TILEDPBUSD_SCALARIZE_ROWS_HEADER]], label [[CONTINUE:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <256 x i32> [[TMP20]] to x86_amx
+; CHECK-NEXT:    store <256 x i32> [[TMP20]], <256 x i32>* [[VPTR:%.*]], align 64
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a.amx = bitcast <256 x i32> %a to x86_amx
+  %b.amx = bitcast <256 x i32> %b to x86_amx
+  %c.amx = bitcast <256 x i32> %c to x86_amx
+  %acc = call x86_amx @llvm.x86.tdpbusd.internal(i16 %row, i16 %col, i16 %k, x86_amx %c.amx, x86_amx %a.amx, x86_amx %b.amx)
+  %vec = bitcast x86_amx %acc to <256 x i32>
+  store <256 x i32> %vec, <256 x i32>* %vptr, align 64
+  ret void
+}
+
+define dso_local void @test_amx_dpbuud(i16 signext %row, i16 signext %col, i16 signext %k, <256 x i32> %c, <256 x i32> %a, <256 x i32> %b, <256 x i32>* %vptr) #0 {
+; CHECK-LABEL: @test_amx_dpbuud(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_AMX:%.*]] = bitcast <256 x i32> [[A:%.*]] to x86_amx
+; CHECK-NEXT:    [[B_AMX:%.*]] = bitcast <256 x i32> [[B:%.*]] to x86_amx
+; CHECK-NEXT:    [[C_AMX:%.*]] = bitcast <256 x i32> [[C:%.*]] to x86_amx
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i16 [[COL:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i16 [[K:%.*]], 2
+; CHECK-NEXT:    br label [[TILEDPBUUD_SCALARIZE_ROWS_HEADER:%.*]]
+; CHECK:       tiledpbuud.scalarize.rows.header:
+; CHECK-NEXT:    [[TILEDPBUUD_SCALARIZE_ROWS_IV:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TILEDPBUUD_SCALARIZE_ROWS_STEP:%.*]], [[TILEDPBUUD_SCALARIZE_ROWS_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_PHI_ROW:%.*]] = phi <256 x i32> [ [[C]], [[ENTRY]] ], [ [[TMP18:%.*]], [[TILEDPBUUD_SCALARIZE_ROWS_LATCH]] ]
+; CHECK-NEXT:    [[VEC_D_PHI_ROW:%.*]] = phi <256 x i32> [ zeroinitializer, [[ENTRY]] ], [ [[TMP20:%.*]], [[TILEDPBUUD_SCALARIZE_ROWS_LATCH]] ]
+; CHECK-NEXT:    br label [[TILEDPBUUD_SCALARIZE_ROWS_BODY:%.*]]
+; CHECK:       tiledpbuud.scalarize.rows.body:
+; CHECK-NEXT:    br label [[TILEDPBUUD_SCALARIZE_COLS_HEADER:%.*]]
+; CHECK:       tiledpbuud.scalarize.cols.header:
+; CHECK-NEXT:    [[TILEDPBUUD_SCALARIZE_COLS_IV:%.*]] = phi i16 [ 0, [[TILEDPBUUD_SCALARIZE_ROWS_BODY]] ], [ [[TILEDPBUUD_SCALARIZE_COLS_STEP:%.*]], [[TILEDPBUUD_SCALARIZE_COLS_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_ROW]], [[TILEDPBUUD_SCALARIZE_ROWS_BODY]] ], [ [[TMP18]], [[TILEDPBUUD_SCALARIZE_COLS_LATCH]] ]
+; CHECK-NEXT:    [[VEC_D_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_D_PHI_ROW]], [[TILEDPBUUD_SCALARIZE_ROWS_BODY]] ], [ [[TMP20]], [[TILEDPBUUD_SCALARIZE_COLS_LATCH]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i16 [[TILEDPBUUD_SCALARIZE_ROWS_IV]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = add i16 [[TMP2]], [[TILEDPBUUD_SCALARIZE_COLS_IV]]
+; CHECK-NEXT:    br label [[TILEDPBUUD_SCALARIZE_COLS_BODY:%.*]]
+; CHECK:       tiledpbuud.scalarize.cols.body:
+; CHECK-NEXT:    br label [[TILEDPBUUD_SCALARIZE_INNER_HEADER:%.*]]
+; CHECK:       tiledpbuud.scalarize.inner.header:
+; CHECK-NEXT:    [[TILEDPBUUD_SCALARIZE_INNER_IV:%.*]] = phi i16 [ 0, [[TILEDPBUUD_SCALARIZE_COLS_BODY]] ], [ [[TILEDPBUUD_SCALARIZE_INNER_STEP:%.*]], [[TILEDPBUUD_SCALARIZE_INNER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_INNER_PHI:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_COL]], [[TILEDPBUUD_SCALARIZE_COLS_BODY]] ], [ [[TMP18]], [[TILEDPBUUD_SCALARIZE_INNER_LATCH]] ]
+; CHECK-NEXT:    br label [[TILEDPBUUD_SCALARIZE_INNER_BODY:%.*]]
+; CHECK:       tiledpbuud.scalarize.inner.body:
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i16 [[TILEDPBUUD_SCALARIZE_ROWS_IV]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = add i16 [[TMP4]], [[TILEDPBUUD_SCALARIZE_INNER_IV]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i16 [[TILEDPBUUD_SCALARIZE_INNER_IV]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = add i16 [[TMP6]], [[TILEDPBUUD_SCALARIZE_COLS_IV]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <256 x i32> [[VEC_C_INNER_PHI]], i16 [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <256 x i32> [[A]], i16 [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP9]] to <4 x i8>
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <256 x i32> [[B]], i16 [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[TMP11]] to <4 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = zext <4 x i8> [[TMP12]] to <4 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = zext <4 x i8> [[TMP10]] to <4 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = mul <4 x i32> [[TMP14]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]])
+; CHECK-NEXT:    [[TMP17:%.*]] = add i32 [[TMP8]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18]] = insertelement <256 x i32> [[VEC_C_INNER_PHI]], i32 [[TMP17]], i16 [[TMP3]]
+; CHECK-NEXT:    br label [[TILEDPBUUD_SCALARIZE_INNER_LATCH]]
+; CHECK:       tiledpbuud.scalarize.inner.latch:
+; CHECK-NEXT:    [[TILEDPBUUD_SCALARIZE_INNER_STEP]] = add i16 [[TILEDPBUUD_SCALARIZE_INNER_IV]], 1
+; CHECK-NEXT:    [[TILEDPBUUD_SCALARIZE_INNER_COND:%.*]] = icmp ne i16 [[TILEDPBUUD_SCALARIZE_INNER_STEP]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[TILEDPBUUD_SCALARIZE_INNER_COND]], label [[TILEDPBUUD_SCALARIZE_INNER_HEADER]], label [[TILEDPBUUD_SCALARIZE_COLS_LATCH]]
+; CHECK:       tiledpbuud.scalarize.cols.latch:
+; CHECK-NEXT:    [[TILEDPBUUD_SCALARIZE_COLS_STEP]] = add i16 [[TILEDPBUUD_SCALARIZE_COLS_IV]], 1
+; CHECK-NEXT:    [[TILEDPBUUD_SCALARIZE_COLS_COND:%.*]] = icmp ne i16 [[TILEDPBUUD_SCALARIZE_COLS_STEP]], [[TMP0]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <256 x i32> [[TMP18]], i16 [[TMP3]]
+; CHECK-NEXT:    [[TMP20]] = insertelement <256 x i32> [[VEC_D_PHI_COL]], i32 [[TMP19]], i16 [[TMP3]]
+; CHECK-NEXT:    br i1 [[TILEDPBUUD_SCALARIZE_COLS_COND]], label [[TILEDPBUUD_SCALARIZE_COLS_HEADER]], label [[TILEDPBUUD_SCALARIZE_ROWS_LATCH]]
+; CHECK:       tiledpbuud.scalarize.rows.latch:
+; CHECK-NEXT:    [[TILEDPBUUD_SCALARIZE_ROWS_STEP]] = add i16 [[TILEDPBUUD_SCALARIZE_ROWS_IV]], 1
+; CHECK-NEXT:    [[TILEDPBUUD_SCALARIZE_ROWS_COND:%.*]] = icmp ne i16 [[TILEDPBUUD_SCALARIZE_ROWS_STEP]], [[ROW:%.*]]
+; CHECK-NEXT:    br i1 [[TILEDPBUUD_SCALARIZE_ROWS_COND]], label [[TILEDPBUUD_SCALARIZE_ROWS_HEADER]], label [[CONTINUE:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <256 x i32> [[TMP20]] to x86_amx
+; CHECK-NEXT:    store <256 x i32> [[TMP20]], <256 x i32>* [[VPTR:%.*]], align 64
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a.amx = bitcast <256 x i32> %a to x86_amx
+  %b.amx = bitcast <256 x i32> %b to x86_amx
+  %c.amx = bitcast <256 x i32> %c to x86_amx
+  %acc = call x86_amx @llvm.x86.tdpbuud.internal(i16 %row, i16 %col, i16 %k, x86_amx %c.amx, x86_amx %a.amx, x86_amx %b.amx)
+  %vec = bitcast x86_amx %acc to <256 x i32>
+  store <256 x i32> %vec, <256 x i32>* %vptr, align 64
+  ret void
+}
+
 define dso_local void @test_amx_dpbf16ps(i16 signext %row, i16 signext %col, i16 signext %k, <256 x i32> %c, <256 x i32> %a, <256 x i32> %b, <256 x i32>* %vptr) #0 {
 ; CHECK-LABEL: @test_amx_dpbf16ps(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A_AMX:%.*]] = bitcast <256 x i32> [[A:%.*]] to x86_amx
 ; CHECK-NEXT:    [[B_AMX:%.*]] = bitcast <256 x i32> [[B:%.*]] to x86_amx
 ; CHECK-NEXT:    [[C_AMX:%.*]] = bitcast <256 x i32> [[C:%.*]] to x86_amx
 ; CHECK-NEXT:    [[TMP0:%.*]] = lshr i16 [[COL:%.*]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr i16 [[K:%.*]], 2
-; CHECK-NEXT:    br label [[TDPBF16PS_SCALARIZE_ROWS_HEADER:%.*]]
-; CHECK:       tdpbf16ps.scalarize.rows.header:
-; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_ROWS_IV:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TDPBF16PS_SCALARIZE_ROWS_STEP:%.*]], [[TDPBF16PS_SCALARIZE_ROWS_LATCH:%.*]] ]
-; CHECK-NEXT:    [[VEC_C_PHI_ROW:%.*]] = phi <256 x i32> [ [[C]], [[ENTRY]] ], [ [[TMP21:%.*]], [[TDPBF16PS_SCALARIZE_ROWS_LATCH]] ]
-; CHECK-NEXT:    [[VEC_D_PHI_ROW:%.*]] = phi <256 x i32> [ zeroinitializer, [[ENTRY]] ], [ [[TMP23:%.*]], [[TDPBF16PS_SCALARIZE_ROWS_LATCH]] ]
-; CHECK-NEXT:    br label [[TDPBF16PS_SCALARIZE_ROWS_BODY:%.*]]
-; CHECK:       tdpbf16ps.scalarize.rows.body:
-; CHECK-NEXT:    br label [[TDPBF16PS_SCALARIZE_COLS_HEADER:%.*]]
-; CHECK:       tdpbf16ps.scalarize.cols.header:
-; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_COLS_IV:%.*]] = phi i16 [ 0, [[TDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TDPBF16PS_SCALARIZE_COLS_STEP:%.*]], [[TDPBF16PS_SCALARIZE_COLS_LATCH:%.*]] ]
-; CHECK-NEXT:    [[VEC_C_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_ROW]], [[TDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TMP21]], [[TDPBF16PS_SCALARIZE_COLS_LATCH]] ]
-; CHECK-NEXT:    [[VEC_D_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_D_PHI_ROW]], [[TDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TMP23]], [[TDPBF16PS_SCALARIZE_COLS_LATCH]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i16 [[TDPBF16PS_SCALARIZE_ROWS_IV]], 16
-; CHECK-NEXT:    [[TMP3:%.*]] = add i16 [[TMP2]], [[TDPBF16PS_SCALARIZE_COLS_IV]]
-; CHECK-NEXT:    br label [[TDPBF16PS_SCALARIZE_COLS_BODY:%.*]]
-; CHECK:       tdpbf16ps.scalarize.cols.body:
-; CHECK-NEXT:    br label [[TDPBF16PS_SCALARIZE_INNER_HEADER:%.*]]
-; CHECK:       tdpbf16ps.scalarize.inner.header:
-; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_INNER_IV:%.*]] = phi i16 [ 0, [[TDPBF16PS_SCALARIZE_COLS_BODY]] ], [ [[TDPBF16PS_SCALARIZE_INNER_STEP:%.*]], [[TDPBF16PS_SCALARIZE_INNER_LATCH:%.*]] ]
-; CHECK-NEXT:    [[VEC_C_INNER_PHI:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_COL]], [[TDPBF16PS_SCALARIZE_COLS_BODY]] ], [ [[TMP21]], [[TDPBF16PS_SCALARIZE_INNER_LATCH]] ]
-; CHECK-NEXT:    br label [[TDPBF16PS_SCALARIZE_INNER_BODY:%.*]]
-; CHECK:       tdpbf16ps.scalarize.inner.body:
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i16 [[TDPBF16PS_SCALARIZE_ROWS_IV]], 16
-; CHECK-NEXT:    [[TMP5:%.*]] = add i16 [[TMP4]], [[TDPBF16PS_SCALARIZE_INNER_IV]]
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i16 [[TDPBF16PS_SCALARIZE_INNER_IV]], 16
-; CHECK-NEXT:    [[TMP7:%.*]] = add i16 [[TMP6]], [[TDPBF16PS_SCALARIZE_COLS_IV]]
+; CHECK-NEXT:    br label [[TILEDPBF16PS_SCALARIZE_ROWS_HEADER:%.*]]
+; CHECK:       tiledpbf16ps.scalarize.rows.header:
+; CHECK-NEXT:    [[TILEDPBF16PS_SCALARIZE_ROWS_IV:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TILEDPBF16PS_SCALARIZE_ROWS_STEP:%.*]], [[TILEDPBF16PS_SCALARIZE_ROWS_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_PHI_ROW:%.*]] = phi <256 x i32> [ [[C]], [[ENTRY]] ], [ [[TMP21:%.*]], [[TILEDPBF16PS_SCALARIZE_ROWS_LATCH]] ]
+; CHECK-NEXT:    [[VEC_D_PHI_ROW:%.*]] = phi <256 x i32> [ zeroinitializer, [[ENTRY]] ], [ [[TMP23:%.*]], [[TILEDPBF16PS_SCALARIZE_ROWS_LATCH]] ]
+; CHECK-NEXT:    br label [[TILEDPBF16PS_SCALARIZE_ROWS_BODY:%.*]]
+; CHECK:       tiledpbf16ps.scalarize.rows.body:
+; CHECK-NEXT:    br label [[TILEDPBF16PS_SCALARIZE_COLS_HEADER:%.*]]
+; CHECK:       tiledpbf16ps.scalarize.cols.header:
+; CHECK-NEXT:    [[TILEDPBF16PS_SCALARIZE_COLS_IV:%.*]] = phi i16 [ 0, [[TILEDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TILEDPBF16PS_SCALARIZE_COLS_STEP:%.*]], [[TILEDPBF16PS_SCALARIZE_COLS_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_ROW]], [[TILEDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TMP21]], [[TILEDPBF16PS_SCALARIZE_COLS_LATCH]] ]
+; CHECK-NEXT:    [[VEC_D_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_D_PHI_ROW]], [[TILEDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TMP23]], [[TILEDPBF16PS_SCALARIZE_COLS_LATCH]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i16 [[TILEDPBF16PS_SCALARIZE_ROWS_IV]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = add i16 [[TMP2]], [[TILEDPBF16PS_SCALARIZE_COLS_IV]]
+; CHECK-NEXT:    br label [[TILEDPBF16PS_SCALARIZE_COLS_BODY:%.*]]
+; CHECK:       tiledpbf16ps.scalarize.cols.body:
+; CHECK-NEXT:    br label [[TILEDPBF16PS_SCALARIZE_INNER_HEADER:%.*]]
+; CHECK:       tiledpbf16ps.scalarize.inner.header:
+; CHECK-NEXT:    [[TILEDPBF16PS_SCALARIZE_INNER_IV:%.*]] = phi i16 [ 0, [[TILEDPBF16PS_SCALARIZE_COLS_BODY]] ], [ [[TILEDPBF16PS_SCALARIZE_INNER_STEP:%.*]], [[TILEDPBF16PS_SCALARIZE_INNER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_INNER_PHI:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_COL]], [[TILEDPBF16PS_SCALARIZE_COLS_BODY]] ], [ [[TMP21]], [[TILEDPBF16PS_SCALARIZE_INNER_LATCH]] ]
+; CHECK-NEXT:    br label [[TILEDPBF16PS_SCALARIZE_INNER_BODY:%.*]]
+; CHECK:       tiledpbf16ps.scalarize.inner.body:
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i16 [[TILEDPBF16PS_SCALARIZE_ROWS_IV]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = add i16 [[TMP4]], [[TILEDPBF16PS_SCALARIZE_INNER_IV]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i16 [[TILEDPBF16PS_SCALARIZE_INNER_IV]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = add i16 [[TMP6]], [[TILEDPBF16PS_SCALARIZE_COLS_IV]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <256 x i32> [[VEC_C_INNER_PHI]], i16 [[TMP3]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32 [[TMP8]] to float
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <256 x i32> [[A]], i16 [[TMP5]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP10]] to <2 x i16>
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <256 x i32> [[B]], i16 [[TMP7]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i32 [[TMP12]] to <2 x i16>
 ; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i16> [[TMP11]], <2 x i16> zeroinitializer, <4 x i32> <i32 2, i32 0, i32 3, i32 1>
 ; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i16> [[TMP14]] to <2 x float>
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x i16> [[TMP13]], <2 x i16> zeroinitializer, <4 x i32> <i32 2, i32 0, i32 3, i32 1>
 ; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <4 x i16> [[TMP16]] to <2 x float>
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <2 x float> [[TMP15]], [[TMP17]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = call float @llvm.vector.reduce.fadd.v2f32(float [[TMP9]], <2 x float> [[TMP18]])
 ; CHECK-NEXT:    [[TMP20:%.*]] = bitcast float [[TMP19]] to i32
 ; CHECK-NEXT:    [[TMP21]] = insertelement <256 x i32> [[VEC_C_INNER_PHI]], i32 [[TMP20]], i16 [[TMP3]]
-; CHECK-NEXT:    br label [[TDPBF16PS_SCALARIZE_INNER_LATCH]]
-; CHECK:       tdpbf16ps.scalarize.inner.latch:
-; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_INNER_STEP]] = add i16 [[TDPBF16PS_SCALARIZE_INNER_IV]], 1
-; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_INNER_COND:%.*]] = icmp ne i16 [[TDPBF16PS_SCALARIZE_INNER_STEP]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[TDPBF16PS_SCALARIZE_INNER_COND]], label [[TDPBF16PS_SCALARIZE_INNER_HEADER]], label [[TDPBF16PS_SCALARIZE_COLS_LATCH]]
-; CHECK:       tdpbf16ps.scalarize.cols.latch:
-; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_COLS_STEP]] = add i16 [[TDPBF16PS_SCALARIZE_COLS_IV]], 1
-; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_COLS_COND:%.*]] = icmp ne i16 [[TDPBF16PS_SCALARIZE_COLS_STEP]], [[TMP0]]
+; CHECK-NEXT:    br label [[TILEDPBF16PS_SCALARIZE_INNER_LATCH]]
+; CHECK:       tiledpbf16ps.scalarize.inner.latch:
+; CHECK-NEXT:    [[TILEDPBF16PS_SCALARIZE_INNER_STEP]] = add i16 [[TILEDPBF16PS_SCALARIZE_INNER_IV]], 1
+; CHECK-NEXT:    [[TILEDPBF16PS_SCALARIZE_INNER_COND:%.*]] = icmp ne i16 [[TILEDPBF16PS_SCALARIZE_INNER_STEP]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[TILEDPBF16PS_SCALARIZE_INNER_COND]], label [[TILEDPBF16PS_SCALARIZE_INNER_HEADER]], label [[TILEDPBF16PS_SCALARIZE_COLS_LATCH]]
+; CHECK:       tiledpbf16ps.scalarize.cols.latch:
+; CHECK-NEXT:    [[TILEDPBF16PS_SCALARIZE_COLS_STEP]] = add i16 [[TILEDPBF16PS_SCALARIZE_COLS_IV]], 1
+; CHECK-NEXT:    [[TILEDPBF16PS_SCALARIZE_COLS_COND:%.*]] = icmp ne i16 [[TILEDPBF16PS_SCALARIZE_COLS_STEP]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <256 x i32> [[TMP21]], i16 [[TMP3]]
 ; CHECK-NEXT:    [[TMP23]] = insertelement <256 x i32> [[VEC_D_PHI_COL]], i32 [[TMP22]], i16 [[TMP3]]
-; CHECK-NEXT:    br i1 [[TDPBF16PS_SCALARIZE_COLS_COND]], label [[TDPBF16PS_SCALARIZE_COLS_HEADER]], label [[TDPBF16PS_SCALARIZE_ROWS_LATCH]]
-; CHECK:       tdpbf16ps.scalarize.rows.latch:
-; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_ROWS_STEP]] = add i16 [[TDPBF16PS_SCALARIZE_ROWS_IV]], 1
-; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_ROWS_COND:%.*]] = icmp ne i16 [[TDPBF16PS_SCALARIZE_ROWS_STEP]], [[ROW:%.*]]
-; CHECK-NEXT:    br i1 [[TDPBF16PS_SCALARIZE_ROWS_COND]], label [[TDPBF16PS_SCALARIZE_ROWS_HEADER]], label [[CONTINUE:%.*]]
+; CHECK-NEXT:    br i1 [[TILEDPBF16PS_SCALARIZE_COLS_COND]], label [[TILEDPBF16PS_SCALARIZE_COLS_HEADER]], label [[TILEDPBF16PS_SCALARIZE_ROWS_LATCH]]
+; CHECK:       tiledpbf16ps.scalarize.rows.latch:
+; CHECK-NEXT:    [[TILEDPBF16PS_SCALARIZE_ROWS_STEP]] = add i16 [[TILEDPBF16PS_SCALARIZE_ROWS_IV]], 1
+; CHECK-NEXT:    [[TILEDPBF16PS_SCALARIZE_ROWS_COND:%.*]] = icmp ne i16 [[TILEDPBF16PS_SCALARIZE_ROWS_STEP]], [[ROW:%.*]]
+; CHECK-NEXT:    br i1 [[TILEDPBF16PS_SCALARIZE_ROWS_COND]], label [[TILEDPBF16PS_SCALARIZE_ROWS_HEADER]], label [[CONTINUE:%.*]]
 ; CHECK:       continue:
 ; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <256 x i32> [[TMP23]] to x86_amx
 ; CHECK-NEXT:    store <256 x i32> [[TMP23]], <256 x i32>* [[VPTR:%.*]], align 64
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %a.amx = bitcast <256 x i32> %a to x86_amx
   %b.amx = bitcast <256 x i32> %b to x86_amx
entry:
   %vec = bitcast x86_amx %amx to <256 x i32>
   store <256 x i32> %vec, <256 x i32>* %vptr, align 64
   ret void
 }
 
 declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
 declare x86_amx @llvm.x86.tileloadd64.internal(i16, i16, i8*, i64)
 declare x86_amx @llvm.x86.tdpbssd.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
+declare x86_amx @llvm.x86.tdpbsud.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
+declare x86_amx @llvm.x86.tdpbusd.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
+declare x86_amx @llvm.x86.tdpbuud.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
 declare x86_amx @llvm.x86.tdpbf16ps.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
 declare void @llvm.x86.tilestored64.internal(i16, i16, i8*, i64, x86_amx)
 
 attributes #0 = { noinline nounwind optnone }