[X86] Pass to transform tdpbf16ps intrinsics to scalar operation.

[yubing]

Summary

In previous patch https://reviews.llvm.org/D93594, we only scalarize tilezero, tileload, tilestore and tiledpbssd. In this patch we scalarize tdpbf16ps intrinsic.

[yubing]

Rebase and add a testcase for dpbf16ps intrinsic.

[pengfei]

llvm/test/CodeGen/X86/AMX/amx-low-intrinsics.ll

174–175 ↗

(On Diff #321675)

Can we use a shuffle instruction?

[yubing]

Address comments above and refactor some code

[yubing]

Rebase and add a testcase.

[yubing]

Fix incorrect naming for dpbf16's bb

[yubing]

Modify some comments

[yubing]

just do a rebase

[pengfei]

llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp

383–385

Is it concise to use below?

template <Intrinsic::ID IntrID>
typename std::enable_if_t<
    IntrID == Intrinsic::x86_tdpbssd_internal ||
    IntrID == Intrinsic::x86_tdpbf16ps_internal, bool>
lowerTileDP(Instruction *TileDP);

389–390

Can we create vecC with <256 x float>?

412

better to use EltCF32 or CF32

419

ditto

420

Better to define a variable for it and reuse.

[yubing]

llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp

389–390

In fact, we are trying to find a bitcast whose operand is <256 x i32>, as shown in line229.

[yubing]

address Pengfei's comments

[pengfei]

LGTM.

[yubing]

Rebase after https://reviews.llvm.org/D98773 is merged.

clang/include/clang/Basic/BuiltinsX86_64.def

 TARGET_BUILTIN(__builtin_ia32_clui, "v", "n", "uintr")
 TARGET_BUILTIN(__builtin_ia32_stui, "v", "n", "uintr")
 TARGET_BUILTIN(__builtin_ia32_testui, "Uc", "n", "uintr")
 TARGET_BUILTIN(__builtin_ia32_senduipi, "vUWi", "n", "uintr")
 
 // AMX internal builtin
 TARGET_BUILTIN(__builtin_ia32_tileloadd64_internal, "V256iUsUsvC*z", "n", "amx-tile")
 TARGET_BUILTIN(__builtin_ia32_tdpbssd_internal, "V256iUsUsUsV256iV256iV256i", "n", "amx-int8")
+TARGET_BUILTIN(__builtin_ia32_tdpbf16ps_internal, "V256iUsUsUsV256iV256iV256i", "n", "amx-bf16")
 TARGET_BUILTIN(__builtin_ia32_tilestored64_internal, "vUsUsv*zV256i", "n", "amx-tile")
 TARGET_BUILTIN(__builtin_ia32_tilezero_internal, "V256iUsUs", "n", "amx-tile")
 // AMX
 TARGET_BUILTIN(__builtin_ia32_tile_loadconfig, "vvC*", "n", "amx-tile")
 TARGET_BUILTIN(__builtin_ia32_tile_storeconfig, "vvC*", "n", "amx-tile")
 TARGET_BUILTIN(__builtin_ia32_tilerelease, "v", "n", "amx-tile")
 TARGET_BUILTIN(__builtin_ia32_tilezero, "vUc", "n", "amx-tile")
 

clang/lib/Headers/amxintrin.h

 /// \param src1
 ///    The 2nd source tile. Max size is 1024 Bytes.
 #define _tile_dpbf16ps(dst, src0, src1)                                        \
   __builtin_ia32_tdpbf16ps((dst), (src0), (src1))
 
 #define __DEFAULT_FN_ATTRS_INT8                                                \
   __attribute__((__always_inline__, __nodebug__, __target__("amx-int8")))
 
+#define __DEFAULT_FN_ATTRS_BF16                                                \
+  __attribute__((__always_inline__, __nodebug__, __target__("amx-bf16")))
+
 typedef int _tile1024i __attribute__((__vector_size__(1024), __aligned__(64)));
 static __inline__ _tile1024i __DEFAULT_FN_ATTRS_INT8
 _tile_loadd_internal(unsigned short m, unsigned short n, const void *base,
                      __SIZE_TYPE__ stride) {
   return __builtin_ia32_tileloadd64_internal(m, n, base,
                                              (__SIZE_TYPE__)(stride));
 }
 
 static __inline__ _tile1024i __DEFAULT_FN_ATTRS_INT8
 _tile_dpbssd_internal(unsigned short m, unsigned short n, unsigned short k,
                       _tile1024i dst, _tile1024i src1, _tile1024i src2) {
   return __builtin_ia32_tdpbssd_internal(m, n, k, dst, src1, src2);
 }
 
+static __inline__ _tile1024i __DEFAULT_FN_ATTRS_INT8
+_tile_dpbf16ps_internal(unsigned short m, unsigned short n, unsigned short k,
+                        _tile1024i dst, _tile1024i src1, _tile1024i src2) {
+  return __builtin_ia32_tdpbf16ps_internal(m, n, k, dst, src1, src2);
+}
+
 static __inline__ void __DEFAULT_FN_ATTRS_INT8
 _tile_stored_internal(unsigned short m, unsigned short n, void *base,
                       __SIZE_TYPE__ stride, _tile1024i tile) {
   return __builtin_ia32_tilestored64_internal(m, n, base,
                                               (__SIZE_TYPE__)(stride), tile);
 }
 
 typedef struct __tile1024i_str {

 
 __DEFAULT_FN_ATTRS_INT8
 static void __tile_dpbssd(__tile1024i *dst, __tile1024i src1,
                           __tile1024i src2) {
   dst->tile = _tile_dpbssd_internal(src1.row, src2.col, src1.col, dst->tile,
                                     src1.tile, src2.tile);
 }
 
+__DEFAULT_FN_ATTRS_INT8
+static void __tile_dpbf16ps(__tile1024i *dst, __tile1024i src1,
+                            __tile1024i src2) {
+  dst->tile = _tile_dpbf16ps_internal(src1.row, src2.col, src1.col, dst->tile,
+                                      src1.tile, src2.tile);
+}
+
 __DEFAULT_FN_ATTRS_TILE
 static void __tile_stored(void *base, __SIZE_TYPE__ stride, __tile1024i src) {
   _tile_stored_internal(src.row, src.col, base, stride, src.tile);
 }
 
 __DEFAULT_FN_ATTRS_TILE
 static void __tile_zero(__tile1024i *dst) {
   dst->tile = __builtin_ia32_tilezero_internal(dst->row, dst->col);
 }
 
 #endif /* __x86_64__ */
 #endif /* __AMXINTRIN_H */

llvm/include/llvm/IR/IntrinsicsX86.td

def int_x86_tileloadd64_internal :
                         [llvm_i16_ty, llvm_i16_ty, llvm_ptr_ty, llvm_i64_ty],
                         []>;
   def int_x86_tdpbssd_internal :
               GCCBuiltin<"__builtin_ia32_tdpbssd_internal">,
               Intrinsic<[llvm_x86amx_ty],
                         [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
                          llvm_x86amx_ty, llvm_x86amx_ty,
                          llvm_x86amx_ty], []>;
+  def int_x86_tdpbf16ps_internal :
+              GCCBuiltin<"__builtin_ia32_tdpbf16ps_internal">,
+              Intrinsic<[llvm_x86amx_ty],
+                        [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
+                         llvm_x86amx_ty, llvm_x86amx_ty,
+                         llvm_x86amx_ty], []>;
   def int_x86_tilestored64_internal :
               GCCBuiltin<"__builtin_ia32_tilestored64_internal">,
               Intrinsic<[], [llvm_i16_ty, llvm_i16_ty, llvm_ptr_ty,
                              llvm_i64_ty, llvm_x86amx_ty], []>;
   def int_x86_tilezero_internal :
               GCCBuiltin<"__builtin_ia32_tilezero_internal">,
               Intrinsic<[llvm_x86amx_ty], [llvm_i16_ty, llvm_i16_ty],
                         []>;

llvm/lib/Target/X86/X86ExpandPseudo.cpp

bool X86ExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
   case X86::PTDPBSSDV: {
     MI.untieRegOperand(4);
     for (unsigned i = 3; i > 0; --i)
       MI.RemoveOperand(i);
     MI.setDesc(TII->get(X86::TDPBSSD));
     MI.tieOperands(0, 1);
     return true;
   }
+  case X86::PTDPBF16PSV: {
+    MI.untieRegOperand(4);
+    for (unsigned i = 3; i > 0; --i)
+      MI.RemoveOperand(i);
+    MI.setDesc(TII->get(X86::TDPBF16PS));
+    MI.tieOperands(0, 1);
+    return true;
+  }
   case X86::PTILESTOREDV: {
     for (int i = 1; i >= 0; --i)
       MI.RemoveOperand(i);
     MI.setDesc(TII->get(X86::TILESTORED));
     return true;
   }
   case X86::PTILEZEROV: {
     for (int i = 2; i > 0; --i) // Remove row, col

llvm/lib/Target/X86/X86ISelDAGToDAG.cpp

case Intrinsic::x86_tdpbssd_internal: {
                        Node->getOperand(6),
                        Node->getOperand(7),
                        Chain};
       MachineSDNode *CNode =
           CurDAG->getMachineNode(Opc, dl, {MVT::x86amx, MVT::Other}, Ops);
       ReplaceNode(Node, CNode);
       return;
     }
+    case Intrinsic::x86_tdpbf16ps_internal: {
+      if (!Subtarget->hasAMXTILE())
+        break;
+      SDValue Chain = Node->getOperand(0);
+      unsigned Opc = X86::PTDPBF16PSV;
+      SDValue Ops[] = {Node->getOperand(2),
+                       Node->getOperand(3),
+                       Node->getOperand(4),
+                       Node->getOperand(5),
+                       Node->getOperand(6),
+                       Node->getOperand(7),
+                       Chain};
+      MachineSDNode *CNode =
+          CurDAG->getMachineNode(Opc, dl, {MVT::x86amx, MVT::Other}, Ops);
+      ReplaceNode(Node, CNode);
+      return;
+    }
     case Intrinsic::x86_tilezero_internal: {
       if (!Subtarget->hasAMXTILE())
         break;
       unsigned Opc = X86::PTILEZEROV;
       SDValue Chain = Node->getOperand(0);
       SDValue Ops[] = {Node->getOperand(2), Node->getOperand(3), Chain};
       MachineSDNode *CNode =
           CurDAG->getMachineNode(Opc, dl, {MVT::x86amx, MVT::Other}, Ops);

llvm/lib/Target/X86/X86InstrAMX.td

let SchedRW = [WriteSystem] in {
     let usesCustomInserter = 1 in {
       // Pseudo instructions, using immediates instead of tile registers.
       // To be translated to the actual instructions in X86ISelLowering.cpp
       def PTDPBF16PS : PseudoI<(outs), (ins u8imm:$src1,
                                u8imm:$src2, u8imm:$src3),
                                [(int_x86_tdpbf16ps timm:$src1,
                                  timm:$src2, timm:$src3)]>;
     }
+    // Pseduo instruction for RA.
+    let Constraints = "$src4 = $dst" in
+    def PTDPBF16PSV : PseudoI<(outs TILE: $dst), (ins GR16:$src1,
+                            GR16:$src2, GR16:$src3, TILE:$src4,
+                            TILE:$src5, TILE:$src6), []>;
   }
 } // HasAMXTILE, HasAMXBF16

llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp

 //
 #include "X86.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/CodeGen/Passes.h"
-
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsX86.h"
static void createTileStoreLoops(BasicBlock *Start, BasicBlock *End,
   // %vec = extractelement <16 x i32> %vec, i16 %idx
   // store i32 %vec, i32* %ptr
   Value *Idx = B.CreateAdd(B.CreateMul(CurrentRow, B.getInt16(16)), CurrentCol);
   Value *Elt = B.CreateExtractElement(Vec, Idx);
 
   B.CreateStore(Elt, EltPtr);
 }
 
-static Value *createTileDPBSSDLoops(BasicBlock *Start, BasicBlock *End,
-                                    IRBuilderBase &B, DomTreeUpdater &DTU,
-                                    LoopInfo &LI, Value *Row, Value *Col,
-                                    Value *K, Value *Acc, Value *LHS,
-                                    Value *RHS) {
+template <Intrinsic::ID IntrID>
+static Value *createTileDPLoops(BasicBlock *Start, BasicBlock *End,
+                                LoopInfo &LI, Value *Row, Value *Col, Value *K,
+                                Value *Acc, Value *LHS, Value *RHS) {
   Loop *RowLoop = LI.AllocateLoop();
   Loop *ColLoop = LI.AllocateLoop();
   Loop *InnerLoop = LI.AllocateLoop();
   ColLoop->addChildLoop(InnerLoop);
   RowLoop->addChildLoop(ColLoop);
   if (Loop *ParentL = LI.getLoopFor(Start))
     ParentL->addChildLoop(RowLoop);
   else
static Value *createTileDPLoops(BasicBlock *Start, BasicBlock *End,
   // i16 %119
   B.SetInsertPoint(InnerBody->getTerminator());
   Value *IdxC =
       B.CreateAdd(B.CreateMul(CurrentRow, B.getInt16(16)), CurrentCol);
   Value *IdxA =
       B.CreateAdd(B.CreateMul(CurrentRow, B.getInt16(16)), CurrentInner);
   Value *IdxB =
       B.CreateAdd(B.CreateMul(CurrentInner, B.getInt16(16)), CurrentCol);
-
+  Value *ResElt = nullptr;
+  if (IntrID == Intrinsic::x86_tdpbssd_internal) {
     FixedVectorType *V4I8Ty = FixedVectorType::get(B.getInt8Ty(), 4);
     FixedVectorType *V4I32Ty = FixedVectorType::get(B.getInt32Ty(), 4);
     Value *EltC = B.CreateExtractElement(VecCPhi, IdxC);
     Value *EltA = B.CreateExtractElement(VecA, IdxA);
     Value *SubVecA = B.CreateBitCast(EltA, V4I8Ty);
     Value *EltB = B.CreateExtractElement(VecB, IdxB);
     Value *SubVecB = B.CreateBitCast(EltB, V4I8Ty);
     Value *SubVecR = B.CreateAddReduce(B.CreateMul(
         B.CreateSExt(SubVecA, V4I32Ty), B.CreateSExt(SubVecB, V4I32Ty)));
-  Value *ResElt = B.CreateAdd(EltC, SubVecR);
+    ResElt = B.CreateAdd(EltC, SubVecR);
+  } else if (IntrID == Intrinsic::x86_tdpbf16ps_internal) {
+    FixedVectorType *V2I16Ty = FixedVectorType::get(B.getInt16Ty(), 2);
+    FixedVectorType *V2F32Ty = FixedVectorType::get(B.getFloatTy(), 2);
+    Value *EltC = B.CreateExtractElement(VecCPhi, IdxC);
+    Value *C_F32 = B.CreateBitCast(EltC, B.getFloatTy());
+    Value *EltA = B.CreateExtractElement(VecA, IdxA);
+    Value *SubVecA = B.CreateBitCast(EltA, V2I16Ty);
+    Value *EltB = B.CreateExtractElement(VecB, IdxB);
+    Value *SubVecB = B.CreateBitCast(EltB, V2I16Ty);
+    Value *ZeroV2I16 = Constant::getNullValue(V2I16Ty);
+    int ShuffleMask[4] = {2, 0, 3, 1};
+    Value *A_V2F32 = B.CreateBitCast(
+        B.CreateShuffleVector(SubVecA, ZeroV2I16, makeArrayRef(ShuffleMask)),
+        V2F32Ty);
+    Value *B_V2F32 = B.CreateBitCast(
+        B.CreateShuffleVector(SubVecB, ZeroV2I16, makeArrayRef(ShuffleMask)),
+        V2F32Ty);
+    Value *SubVecR = B.CreateFAddReduce(C_F32, B.CreateFMul(A_V2F32, B_V2F32));
+    ResElt = B.CreateBitCast(SubVecR, B.getInt32Ty());
+  } else {
+    llvm_unreachable("it is not a tdpb intrinsic");
+  }
   Value *NewVecC = B.CreateInsertElement(VecCPhi, ResElt, IdxC);
   Value *NewVecD = B.CreateInsertElement(VecDPhi, ResElt, IdxC);
 
   VecCPhi->addIncoming(NewVecC, InnerLoopLatch);
   VecCPhiRowLoop->addIncoming(NewVecC, RowLatch);
   VecCPhiColLoop->addIncoming(NewVecC, ColLoopLatch);
   VecDPhi->addIncoming(NewVecD, InnerLoopLatch);
   VecDPhiRowLoop->addIncoming(NewVecD, RowLatch);
public:
   X86LowerAMXIntrinsics(Function &F, DominatorTree *DT, LoopInfo *LI)
       : Func(F), DT(DT), LI(LI) {}
   bool visit();
 
 private:
   DominatorTree *DT;
   LoopInfo *LI;
   bool lowerTileLoad(Instruction *TileLoad);
-  bool lowerTileDPBSSD(Instruction *TileDPBSSD);
+  template <Intrinsic::ID IntrID> bool lowerTileDP(Instruction *TileDP);
   bool lowerTileStore(Instruction *TileStore);
   bool lowerTileZero(Instruction *TileZero);
 };
 
-bool X86LowerAMXIntrinsics::lowerTileDPBSSD(Instruction *TileDPBSSD) {
+template <Intrinsic::ID IntrID>
+bool X86LowerAMXIntrinsics::lowerTileDP(Instruction *TileDP) {
   Value *M, *N, *K, *C, *A, *B;
-  match(TileDPBSSD, m_Intrinsic<Intrinsic::x86_tdpbssd_internal>(
-                        m_Value(M), m_Value(N), m_Value(K), m_Value(C),
-                        m_Value(A), m_Value(B)));
+  match(TileDP, m_Intrinsic<IntrID>(m_Value(M), m_Value(N), m_Value(K),
+                                    m_Value(C), m_Value(A), m_Value(B)));
   DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
-  Instruction *InsertI = TileDPBSSD;
-  IRBuilder<> BuilderPrepare(TileDPBSSD);
-  BuilderPrepare.SetInsertPoint(TileDPBSSD);
+  Instruction *InsertI = TileDP;
+  IRBuilder<> BuilderPrepare(TileDP);
+  BuilderPrepare.SetInsertPoint(TileDP);
   // We visit the loop with (m, n/4, k/4):
   // %n_dword = udiv i16 %n, 4
   // %k_dword = udiv i16 %k, 4
   Value *NDWord = BuilderPrepare.CreateUDiv(N, BuilderPrepare.getInt16(4));
   Value *KDWord = BuilderPrepare.CreateUDiv(K, BuilderPrepare.getInt16(4));
   BasicBlock *Start = InsertI->getParent();
   BasicBlock *End =
       SplitBlock(InsertI->getParent(), InsertI, DT, LI, nullptr, "continue");
-  IRBuilder<> Builder(TileDPBSSD);
-  Value *ResVec = createTileDPBSSDLoops(Start, End, Builder, DTU, *LI, M,
+  IRBuilder<> Builder(TileDP);
+  Value *ResVec = createTileDPLoops<IntrID>(Start, End, Builder, DTU, *LI, M,
                                             NDWord, KDWord, C, A, B);
-  // we cannot assume there always be bitcast after tiledpbssd. So we need to
+  // we cannot assume there always be bitcast after TileDP. So we need to
   // insert one bitcast as required
   Builder.SetInsertPoint(End->getFirstNonPHI());
   Value *ResAMX =
       Builder.CreateBitCast(ResVec, Type::getX86_AMXTy(Builder.getContext()));
-  // Delete tiledpbssd intrinsic and do some clean-up.
-  for (auto UI = TileDPBSSD->use_begin(), UE = TileDPBSSD->use_end();
-       UI != UE;) {
+  // Delete TileDP intrinsic and do some clean-up.
+  for (auto UI = TileDP->use_begin(), UE = TileDP->use_end(); UI != UE;) {
     Instruction *I = cast<Instruction>((UI++)->getUser());
     Value *Vec;
     if (match(I, m_BitCast(m_Value(Vec)))) {
       I->replaceAllUsesWith(ResVec);
       I->eraseFromParent();
     }
   }
-  TileDPBSSD->replaceAllUsesWith(ResAMX);
-  TileDPBSSD->eraseFromParent();
+  TileDP->replaceAllUsesWith(ResAMX);
+  TileDP->eraseFromParent();
   return true;
 }
 
 bool X86LowerAMXIntrinsics::lowerTileLoad(Instruction *TileLoad) {
   Value *M, *N, *Ptr, *Stride;
   match(TileLoad, m_Intrinsic<Intrinsic::x86_tileloadd64_internal>(
                       m_Value(M), m_Value(N), m_Value(Ptr), m_Value(Stride)));
   DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
bool X86LowerAMXIntrinsics::lowerTileZero(Instruction *TileZero) {
   }
   TileZero->eraseFromParent();
   return true;
 }
 
 bool X86LowerAMXIntrinsics::visit() {
   bool C = false;
   SmallVector<Instruction *, 8> TileDPBSSDs;
+  SmallVector<Instruction *, 8> TileDPBF16PSs;
   SmallVector<Instruction *, 8> TileLoads;
   SmallVector<Instruction *, 8> TileStores;
   SmallVector<Instruction *, 8> TileZeros;
   SmallVector<Instruction *, 8> WorkList;
   for (BasicBlock *BB : depth_first(&Func)) {
     for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) {
       Instruction &Inst = *II++;
       if (match(&Inst, m_Intrinsic<Intrinsic::x86_tdpbssd_internal>()) ||
+          match(&Inst, m_Intrinsic<Intrinsic::x86_tdpbf16ps_internal>()) ||
           match(&Inst, m_Intrinsic<Intrinsic::x86_tileloadd64_internal>()) ||
           match(&Inst, m_Intrinsic<Intrinsic::x86_tilestored64_internal>()) ||
           match(&Inst, m_Intrinsic<Intrinsic::x86_tilezero_internal>()))
         // %89 = bitcast <256 x i32> %88 to x86_amx
         // call void @llvm.x86.tilezero.internal(i16 %84, i16 %85)
         WorkList.push_back(&Inst);
     }
   }
 
   for (auto *Inst : WorkList) {
     if (match(Inst, m_Intrinsic<Intrinsic::x86_tdpbssd_internal>()))
       // %amx1 = bitcast <256 x i32> %vec to x86_amx
       // %res = call x86_amx @llvm.x86.tdpbssd.internal(i16 m, i16 n, i16 k,
       //                                                x86_amx, %amx1, ...)
       // %vec2 = bitcast x86_amx %res to <256 x i32>
-      C = lowerTileDPBSSD(Inst) || C;
+      C = lowerTileDP<Intrinsic::x86_tdpbssd_internal>(Inst) || C;
+    else if (match(Inst, m_Intrinsic<Intrinsic::x86_tdpbf16ps_internal>()))
+      // %amx1 = bitcast <256 x i32> %vec to x86_amx
+      // %res = call x86_amx @llvm.x86.tdpbf16ps.internal(i16 m, i16 n, i16 k,
+      //                                                x86_amx, %amx1, ...)
+      // %vec2 = bitcast x86_amx %res to <256 x i32>
+      C = lowerTileDP<Intrinsic::x86_tdpbf16ps_internal>(Inst) || C;
     else if (match(Inst, m_Intrinsic<Intrinsic::x86_tileloadd64_internal>()))
       // %17 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %13, i16 %14,
       //                                                   i8* %15, i64 %16)
       // %18 = bitcast x86_amx %17 to <256 x i32>
       C = lowerTileLoad(Inst) || C;
     else if (match(Inst, m_Intrinsic<Intrinsic::x86_tilestored64_internal>()))
       // %89 = bitcast <256 x i32> %88 to x86_amx
       // call void @llvm.x86.tilestored64.internal(i16 %84, i16 %85, i8* %86,

llvm/lib/Target/X86/X86LowerAMXType.cpp

static std::pair<Value *, Value *> getShape(IntrinsicInst *II, unsigned OpNo) {
   case Intrinsic::x86_tileloadd64_internal:
   case Intrinsic::x86_tilestored64_internal: {
     Row = II->getArgOperand(0);
     Col = II->getArgOperand(1);
     break;
   }
   // a * b + c
   // The shape depends on which operand.
-  case Intrinsic::x86_tdpbssd_internal: {
+  case Intrinsic::x86_tdpbssd_internal:
+  case Intrinsic::x86_tdpbf16ps_internal: {
     switch (OpNo) {
     case 3:
       Row = II->getArgOperand(0);
       Col = II->getArgOperand(1);
       break;
     case 4:
       Row = II->getArgOperand(0);
       Col = II->getArgOperand(2);

llvm/lib/Target/X86/X86PreTileConfig.cpp

 
 static ShapeT getShape(const MachineInstr &MI, MachineRegisterInfo *MRI) {
   unsigned Opcode = MI.getOpcode();
   switch (Opcode) {
   default:
     llvm_unreachable("Unexpected machine instruction on tile");
   case X86::PTILELOADDV:
   case X86::PTDPBSSDV:
+  case X86::PTDPBF16PSV:
   case X86::PTILEZEROV:
     MachineOperand &MO1 = const_cast<MachineOperand &>(MI.getOperand(1));
     MachineOperand &MO2 = const_cast<MachineOperand &>(MI.getOperand(2));
     ShapeT Shape(&MO1, &MO2, MRI);
     return Shape;
   }
 }
 

 
 static bool isAMXInstruction(MachineBasicBlock::iterator MII) {
   switch (MII->getOpcode()) {
   default:
     return false;
   case X86::PTILELOADDV:
   case X86::PTILESTOREDV:
   case X86::PTDPBSSDV:
+  case X86::PTDPBF16PSV:
   case X86::PTILEZEROV:
     return true;
   }
 }
 
 struct BBInfo {
   bool HasAMX = false;
   bool HasCallBeforeAMX = false;

llvm/lib/Target/X86/X86RegisterInfo.cpp

static ShapeT getTileShape(Register VirtReg, VirtRegMap *VRM,
   unsigned OpCode = MI->getOpcode();
   switch (OpCode) {
   default:
     llvm_unreachable("Unexpected machine instruction on tile register!");
     break;
   // We only collect the tile shape that is defined.
   case X86::PTILELOADDV:
   case X86::PTDPBSSDV:
+  case X86::PTDPBF16PSV:
   case X86::PTILEZEROV:
     MachineOperand &MO1 = MI->getOperand(1);
     MachineOperand &MO2 = MI->getOperand(2);
     ShapeT Shape(&MO1, &MO2, MRI);
     VRM->assignVirt2Shape(VirtReg, Shape);
     return Shape;
   }
 }