[X86][AMX] Refactor for PostRA ldtilecfg pass.

[pengfei]

Summary

This is a follow up of D99010. We didn't consider the live range of shape registers when hoist ldtilecfg. There maybe risks, e.g. we happen to insert it to an invalid range of some registers and get unexpected error.

This patch fixes this problem by storing the value to corresponding stack place of ldtilecfg after all its definition immediately.

This patch also fix a problem in previous code: If we don't have a ldtilecfg which dominates all AMX instructions, we cannot initialize shapes for other ldtilecfg.

There're still some optimization points left. E.g. eliminate unused mov instructions, break the def-use dependency before RA etc.

[pengfei]

A workable implementation.

[pengfei]

Add comments for shape layout.

[pengfei]

Rebase.

[xiangzhangllvm]

llvm/lib/Target/X86/X86TileConfig.cpp

153

The problem is the DefMI may comes from a reload, at its live range may not include the "shape_config_write" instructions which you want generated.

[pengfei]

llvm/lib/Target/X86/X86TileConfig.cpp

153

We insert MOV8mi/MOV8mr adjoining any DefMI, so live range is not a problem.
We may insert redundant store instructions for reload, but it won't result in correctness problems.

[xiangzhangllvm]

llvm/lib/Target/X86/X86TileConfig.cpp

123

I see here suppose all virtual tile regs are same shape, how we handle the cfg for tileload/store at spill ?

[pengfei]

llvm/lib/Target/X86/X86TileConfig.cpp

123

The assumption is all virtual regs that bound to the same physical regs are the same shape. We don't handle cfg when spill.

[xiangzhangllvm]

LGTM, (spill will only happen in the same shape tile register.)

[LuoYuanke]

llvm/lib/Target/X86/X86TileConfig.cpp

111

Would you add comments to explain in PreRA we have inserted palette id in the entry?

[pengfei]

Address Yuanke's comment.

[LuoYuanke]

LGTM. Thank you!

llvm/lib/Target/X86/X86PreTileConfig.cpp

bool operator>(const MIRef &RHS) const {
     return Pos > RHS.Pos;
   }
 };
 
 struct BBInfo {
   MIRef FirstAMX;
   MIRef LastCall;
   MIRef LastShape;
+  bool TileCfgForbidden = false;
   bool NeedTileCfgLiveIn = false;
-  unsigned ShapeReachedCount = 0;
 };
 
 class X86PreTileConfig : public MachineFunctionPass {
   MachineRegisterInfo *MRI;
   const MachineLoopInfo *MLI;
   SmallSet<MachineInstr *, 8> DefVisited;
   SmallSet<MachineBasicBlock *, 8> ShapeBBs;
   DenseMap<MachineBasicBlock *, BBInfo> BBVisitedInfo;
for (auto *Pred : MBB->predecessors()) {
       }
     }
   }
 
   // There's no AMX instruction if we didn't find a tile config live in point.
   if (CfgNeedInsert.empty())
     return false;
 
-  // Calculate how many times the ShapeBB can reach to this BB.
-  unsigned ShapeBBNum = 0;
-  for (auto *MBB : ShapeBBs) {
-    SmallSet<MachineBasicBlock *, 8> VistedBB;
-    SmallVector<MachineBasicBlock *, 8> WorkList({MBB});
-    while (!WorkList.empty()) {
-      MachineBasicBlock *MBB = WorkList.pop_back_val();
-      ++BBVisitedInfo[MBB].ShapeReachedCount;
-      for (auto *Succ : MBB->successors())
-        if (VistedBB.insert(Succ).second && !isLoopBackEdge(Succ, MBB))
-          WorkList.push_back(Succ);
+  // Avoid to insert ldtilecfg before any shape defs.
+  SmallVector<MachineBasicBlock *, 8> WorkList(
+      make_range(ShapeBBs.begin(), ShapeBBs.end()));
+    for (auto *Pred : MBB->predecessors()) {
+      if (!BBVisitedInfo[Pred].TileCfgForbidden && !isLoopBackEdge(MBB, Pred)) {
+        BBVisitedInfo[Pred].TileCfgForbidden = true;
+        WorkList.push_back(Pred);
+      }
     }
-    ++ShapeBBNum;
   }
 
   DebugLoc DL;
   SmallSet<MIRef, 8> VisitedOrInserted;
   int SS = MF.getFrameInfo().CreateStackObject(
       ST.getTileConfigSize(), ST.getTileConfigAlignment(), false);
 
   // Try to insert for the tile config live in points.
   for (auto I : CfgNeedInsert) {
     SmallSet<MIRef, 8> InsertPoints;
     SmallVector<MIRef, 8> WorkList({I});
     while (!WorkList.empty()) {
       MIRef I = WorkList.pop_back_val();
       if (!VisitedOrInserted.count(I)) {
-        if (BBVisitedInfo[I.MBB].ShapeReachedCount == ShapeBBNum) {
+        if (!BBVisitedInfo[I.MBB].TileCfgForbidden) {
           // If the BB is all shapes reachable, stop sink and try to insert.
           InsertPoints.insert(I);
         } else {
           // Avoid the BB to be multi visited.
           VisitedOrInserted.insert(I);
           // We cannot sink it across any AMX instruction.
           if (BBVisitedInfo[I.MBB].FirstAMX)
             REPORT_CONFIG_FAIL;
addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOVUPSmr)), SS)
         .addReg(Xmm);
     addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOVUPSmr)), SS, 16)
         .addReg(Xmm);
     addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOVUPSmr)), SS, 32)
         .addReg(Xmm);
     addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOVUPSmr)), SS, 48)
         .addReg(Xmm);
   }
+  // Fill in the palette first.
+  addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV8mi)), SS).addImm(1);
 
   return true;
 }
 
 FunctionPass *llvm::createX86PreTileConfigPass() {
   return new X86PreTileConfig();
 }

llvm/lib/Target/X86/X86TileConfig.cpp

 //
 //===----------------------------------------------------------------------===//
 
 #include "X86.h"
 #include "X86InstrBuilder.h"
 #include "X86MachineFunctionInfo.h"
 #include "X86RegisterInfo.h"
 #include "X86Subtarget.h"
-#include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/CodeGen/LiveIntervals.h"
-#include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TileShapeInfo.h"
 #include "llvm/CodeGen/VirtRegMap.h"
 #include "llvm/InitializePasses.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "tile-config"
 
 namespace {
 
-class X86TileConfig : public MachineFunctionPass {
-  // context
-  MachineFunction *MF = nullptr;
-  const X86Subtarget *ST = nullptr;
-  const TargetRegisterInfo *TRI;
-  const TargetInstrInfo *TII;
-  MachineDominatorTree *DomTree = nullptr;
-  MachineRegisterInfo *MRI = nullptr;
-  VirtRegMap *VRM = nullptr;
-  LiveIntervals *LIS = nullptr;
+struct X86TileConfig : public MachineFunctionPass {
 
-  MachineInstr *getTileConfigPoint();
-  void tileConfig();
-
-public:
   X86TileConfig() : MachineFunctionPass(ID) {}
 
   /// Return the pass name.
   StringRef getPassName() const override { return "Tile Register Configure"; }
 
   /// X86TileConfig analysis usage.
-  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+    AU.addRequired<VirtRegMap>();
+    AU.addRequired<LiveIntervals>();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
 
   /// Perform register allocation.
   bool runOnMachineFunction(MachineFunction &mf) override;
 
   MachineFunctionProperties getRequiredProperties() const override {
     return MachineFunctionProperties().set(
         MachineFunctionProperties::Property::NoPHIs);
   }
 
   static char ID;
 };
 
 } // end anonymous namespace
 
 char X86TileConfig::ID = 0;
 
 INITIALIZE_PASS_BEGIN(X86TileConfig, "tileconfig", "Tile Register Configure",
                       false, false)
-INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
 INITIALIZE_PASS_DEPENDENCY(VirtRegMap)
 INITIALIZE_PASS_END(X86TileConfig, "tileconfig", "Tile Register Configure",
                     false, false)
 
-void X86TileConfig::getAnalysisUsage(AnalysisUsage &AU) const {
-  AU.addRequired<MachineDominatorTree>();
-  AU.addRequired<LiveIntervals>();
-  AU.addPreserved<SlotIndexes>();
-  AU.addRequired<VirtRegMap>();
-  AU.setPreservesAll();
-  MachineFunctionPass::getAnalysisUsage(AU);
+bool X86TileConfig::runOnMachineFunction(MachineFunction &MF) {
+  const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
+  const TargetRegisterInfo *TRI = ST.getRegisterInfo();
+  const TargetInstrInfo *TII = ST.getInstrInfo();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+  LiveIntervals &LIS = getAnalysis<LiveIntervals>();
+  VirtRegMap &VRM = getAnalysis<VirtRegMap>();
+
+  if (VRM.isShapeMapEmpty())
+    return false;
+
+  int SS = INT_MAX;
+  for (MachineBasicBlock &MBB : MF) {
+    for (MachineInstr &MI : MBB) {
+      if (MI.getOpcode() == X86::LDTILECFG) {
+        SS = MI.getOperand(0).getIndex();
+        break;
+      }
+    }
+    if (SS != INT_MAX)
+      break;
   }
 
-static unsigned getTilePhysRegIndex(Register PhysReg) {
-  assert((PhysReg >= X86::TMM0 && X86::TMM0 <= X86::TMM7) &&
-         "Tile register number is invalid");
-  return (PhysReg - X86::TMM0);
+  unsigned ConstPos = 0;
+  MachineInstr *ConstMI = nullptr;
+  for (MachineInstr &MI : MF.front()) {
+    if (MI.getOpcode() == X86::MOV8mi && SS == MI.getOperand(0).getIndex()) {
+      ConstMI = &MI;
+      break;
     }
+    ++ConstPos;
+  }
+  assert(ConstMI && "Cannot find a insertion point");
 
-static MachineInstr *
-storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
-                    Register SrcReg, unsigned BitSize, int FrameIdx, int Offset,
-                    const TargetInstrInfo *TII, const TargetRegisterClass *RC,
-                    const TargetRegisterInfo *TRI) {
-
-  unsigned SubIdx = (BitSize == 8) ? X86::sub_8bit : X86::sub_16bit;
-  unsigned Opc = (BitSize == 8) ? X86::MOV8mr : X86::MOV16mr;
-  if (BitSize == TRI->getRegSizeInBits(*RC))
-    SubIdx = 0;
-  MachineInstr *NewMI =
-      addFrameReference(BuildMI(MBB, MI, DebugLoc(), TII->get(Opc)), FrameIdx,
-                        Offset)
-          .addReg(SrcReg, 0, SubIdx);
-  return NewMI;
-}
-
-static MachineInstr *storeImmToStackSlot(MachineBasicBlock &MBB,
-                                         MachineBasicBlock::iterator MI,
-                                         int64_t Imm, unsigned BitSize,
-                                         int FrameIdx, int Offset,
-                                         const TargetInstrInfo *TII) {
-  unsigned Opc = (BitSize == 8) ? X86::MOV8mi : X86::MOV16mi;
-  return addFrameReference(BuildMI(MBB, MI, DebugLoc(), TII->get(Opc)),
-                           FrameIdx, Offset)
-      .addImm(Imm);
-}
-
-MachineInstr *X86TileConfig::getTileConfigPoint() {
-  MachineBasicBlock *Entry = &*MF->begin();
-  ReversePostOrderTraversal<MachineBasicBlock *> RPOT(Entry);
-  for (MachineBasicBlock *MBB : RPOT) {
-    for (MachineInstr &MI : *MBB)
-      // Refer X86PreTileConfig.cpp.
-      // We only support one tile config for now. The other ldtilecfg
-      // is for spill purpose and is dominated by the first ldtilecfg.
-      if (MI.getOpcode() == X86::LDTILECFG)
-        return &MI;
-  }
-
-  return nullptr;
-}
-
-void X86TileConfig::tileConfig() {
-  MachineInstr *MI = getTileConfigPoint();
-  if (!MI)
-    return;
-  MachineBasicBlock *MBB = MI->getParent();
-  int SS = MI->getOperand(0).getIndex();
-  BitVector PhysRegs(TRI->getNumRegs());
-
-  // Fill in the palette first.
-  auto *NewMI = storeImmToStackSlot(*MBB, *MI, 1, 8, SS, 0, TII);
-  LIS->InsertMachineInstrInMaps(*NewMI);
-  // Fill in the shape of each tile physical register.
-  for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
-    Register VirtReg = Register::index2VirtReg(i);
-    if (MRI->reg_nodbg_empty(VirtReg))
+  const TargetRegisterClass *RC = TRI->getRegClass(X86::TILERegClassID);
+  SmallVector<Register, 8> Phys2Virt(RC->getNumRegs(), 0);
+  for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
+    Register VirtReg = Register::index2VirtReg(I);
+    if (MRI.reg_nodbg_empty(VirtReg))
       continue;
-    const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
+    const TargetRegisterClass &RC = *MRI.getRegClass(VirtReg);
     if (RC.getID() != X86::TILERegClassID)
       continue;
-    Register PhysReg = VRM->getPhys(VirtReg);
-    if (PhysRegs.test(PhysReg))
-      continue;
-    PhysRegs.set(PhysReg);
-    ShapeT Shape = VRM->getShape(VirtReg);
-    Register RowReg = Shape.getRow()->getReg();
-    Register ColReg = Shape.getCol()->getReg();
-
-    // Here is the data format for the tile config.
-    // 0      palette
-    // 1      start_row
-    // 2-15   reserved, must be zero
-    // 16-17  tile0.colsb Tile 0 bytes per row.
-    // 18-19  tile1.colsb Tile 1 bytes per row.
-    // 20-21  tile2.colsb Tile 2 bytes per row.
-    // ... (sequence continues)
-    // 30-31  tile7.colsb Tile 7 bytes per row.
-    // 32-47  reserved, must be zero
-    // 48     tile0.rows Tile 0 rows.
-    // 49     tile1.rows Tile 1 rows.
-    // 50     tile2.rows Tile 2 rows.
-    // ... (sequence continues)
-    // 55     tile7.rows Tile 7 rows.
-    // 56-63  reserved, must be zero
-    unsigned Index = getTilePhysRegIndex(PhysReg);
-    int RowOffset = 48 + Index;
-    int ColOffset = 16 + Index * 2;
-
-    unsigned BitSize = 8;
-    for (const auto &Pair : {std::make_pair(RowReg, RowOffset),
-                             std::make_pair(ColReg, ColOffset)}) {
-      int64_t Imm;
-      int ImmCount = 0;
-      // All def must be the same value, otherwise it is invalid MIs.
-      // Immediate is prefered.
-      for (const MachineOperand &MO : MRI->def_operands(Pair.first)) {
-        const auto *Inst = MO.getParent();
-        if (Inst->isMoveImmediate()) {
-          ImmCount++;
-          Imm = Inst->getOperand(1).getImm();
-          break;
-        }
-      }
-      auto StoreConfig = [&](int Offset) {
-        MachineInstr *NewMI = nullptr;
-        if (ImmCount)
-          NewMI = storeImmToStackSlot(*MBB, *MI, Imm, BitSize, SS, Offset, TII);
-        else {
-          const TargetRegisterClass *RC = MRI->getRegClass(Pair.first);
-          NewMI = storeRegToStackSlot(*MBB, *MI, Pair.first, BitSize, SS,
-                                      Offset, TII, RC, TRI);
-        }
-        SlotIndex SIdx = LIS->InsertMachineInstrInMaps(*NewMI);
-        if (!ImmCount) {
-          // Extend the live interval.
-          SmallVector<SlotIndex, 8> EndPoints = {SIdx.getRegSlot()};
-          LiveInterval &Int = LIS->getInterval(Pair.first);
-          LIS->extendToIndices(Int, EndPoints);
+    unsigned Index = VRM.getPhys(VirtReg) - X86::TMM0;
+    if (!Phys2Virt[Index])
+      Phys2Virt[Index] = VirtReg;
+
+  // Fill in the shape of each tile physical register.
+  for (unsigned I = 0, E = RC->getNumRegs(); I != E; ++I) {
+    if (!Phys2Virt[I])
+      continue;
+    DebugLoc DL;
+    bool IsRow = true;
+    ShapeT Shape = VRM.getShape(Phys2Virt[I]);
+    for (auto &R : {Shape.getRow()->getReg(), Shape.getCol()->getReg()}) {
+      int Offset = IsRow ? 48 + I : 16 + I * 2;
+      for (auto &DefMI : MRI.def_instructions(R)) {
+        int ImmCount = 0;
+        MachineBasicBlock &MBB = *DefMI.getParent();
+        if (DefMI.isMoveImmediate()) {
+          // FIXME: We should handle this case in future.
+          assert(++ImmCount == 1 && "Cannot initialize with different shapes");
+          NewMI = addFrameReference(
+                      BuildMI(MF.front(), ++ConstMI->getIterator(), DL,
+                              TII->get(IsRow ? X86::MOV8mi : X86::MOV16mi)),
+                      SS, Offset)
+                      .addImm(DefMI.getOperand(1).getImm());
+          ConstMI = NewMI;
+          LIS.InsertMachineInstrInMaps(*NewMI);
+        } else {
+          unsigned SubIdx = IsRow ? X86::sub_8bit : X86::sub_16bit;
+          unsigned RegSize = TRI->getRegSizeInBits(*MRI.getRegClass(R));
+          if ((IsRow && RegSize == 8) || (!IsRow && RegSize == 16))
+            SubIdx = 0;
+          auto Iter = DefMI.getIterator();
+          if (&MBB == &MF.front() &&
+              std::distance(MBB.instr_begin(), Iter) < ConstPos)
+            Iter = ConstMI->getIterator();
+          NewMI = addFrameReference(
+                      BuildMI(MBB, ++Iter, DL,
+                              TII->get(IsRow ? X86::MOV8mr : X86::MOV16mr)),
+                      SS, Offset)
+                      .addReg(R, 0, SubIdx);
+          SlotIndex SIdx = LIS.InsertMachineInstrInMaps(*NewMI);
+          LIS.extendToIndices(LIS.getInterval(R), {SIdx.getRegSlot()});
         }
-      };
-      StoreConfig(Pair.second);
-      BitSize += 8;
       }
+      IsRow = false;
     }
   }
-
-bool X86TileConfig::runOnMachineFunction(MachineFunction &mf) {
-  MF = &mf;
-  MRI = &mf.getRegInfo();
-  ST = &mf.getSubtarget<X86Subtarget>();
-  TRI = ST->getRegisterInfo();
-  TII = mf.getSubtarget().getInstrInfo();
-  DomTree = &getAnalysis<MachineDominatorTree>();
-  VRM = &getAnalysis<VirtRegMap>();
-  LIS = &getAnalysis<LiveIntervals>();
-
-  if (VRM->isShapeMapEmpty())
-    return false;
-
-  tileConfig();
   return true;
 }
 
 FunctionPass *llvm::createX86TileConfigPass() { return new X86TileConfig(); }

llvm/test/CodeGen/X86/AMX/amx-across-func.ll

 ; CHECK-NEXT:    pushq %r14
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    subq $4056, %rsp # imm = 0xFD8
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
 ; CHECK-NEXT:    vmovdqu64 %zmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb $1, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movw %bx, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw $8, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb $8, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %bx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %bx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    ldtilecfg {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movl $buf, %eax
 ; CHECK-NEXT:    movl $32, %r14d
 ; CHECK-NEXT:    movw $8, %r15w
 ; CHECK-NEXT:    tileloadd (%rax,%r14), %tmm1
 ; CHECK-NEXT:    movabsq $64, %rax
 ; CHECK-NEXT:    tilestored %tmm1, 2048(%rsp,%rax) # 1024-byte Folded Spill
 ; CHECK-NEXT:    movl $buf+1024, %eax

 ; CHECK-NEXT:    retq
 ;
 ; IPRA-LABEL: test_api:
 ; IPRA:       # %bb.0:
 ; IPRA-NEXT:    subq $72, %rsp
 ; IPRA-NEXT:    vpxord %zmm0, %zmm0, %zmm0
 ; IPRA-NEXT:    vmovdqu64 %zmm0, {{[0-9]+}}(%rsp)
 ; IPRA-NEXT:    movb $1, {{[0-9]+}}(%rsp)
-; IPRA-NEXT:    movb %dil, {{[0-9]+}}(%rsp)
 ; IPRA-NEXT:    movw $8, {{[0-9]+}}(%rsp)
 ; IPRA-NEXT:    movb $8, {{[0-9]+}}(%rsp)
 ; IPRA-NEXT:    movw %si, {{[0-9]+}}(%rsp)
 ; IPRA-NEXT:    movb %dil, {{[0-9]+}}(%rsp)
 ; IPRA-NEXT:    movw %si, {{[0-9]+}}(%rsp)
+; IPRA-NEXT:    movb %dil, {{[0-9]+}}(%rsp)
 ; IPRA-NEXT:    ldtilecfg {{[0-9]+}}(%rsp)
 ; IPRA-NEXT:    movl $buf, %eax
 ; IPRA-NEXT:    movl $32, %ecx
 ; IPRA-NEXT:    movw $8, %dx
 ; IPRA-NEXT:    tileloadd (%rax,%rcx), %tmm0
 ; IPRA-NEXT:    movl $buf+1024, %eax
 ; IPRA-NEXT:    tileloadd (%rax,%rcx), %tmm1
 ; IPRA-NEXT:    callq foo

 ; CHECK-NEXT:    pushq %r14
 ; CHECK-NEXT:    pushq %r13
 ; CHECK-NEXT:    pushq %r12
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    subq $3016, %rsp # imm = 0xBC8
 ; CHECK-NEXT:    movl %edi, %r14d
 ; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
 ; CHECK-NEXT:    vmovdqu64 %zmm0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    callq foo
 ; CHECK-NEXT:    movb $1, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb $8, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw $8, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq foo
 ; CHECK-NEXT:    ldtilecfg {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    testl %r14d, %r14d
 ; CHECK-NEXT:    jg .LBB2_4
 ; CHECK-NEXT:  # %bb.1: # %.preheader
 ; CHECK-NEXT:    movl $7, %ebp
 ; CHECK-NEXT:    movl $buf, %r15d
 ; CHECK-NEXT:    movl $32, %r12d
 ; CHECK-NEXT:    movw $8, %bx

 ; CHECK-NEXT:    pushq %r15
 ; CHECK-NEXT:    pushq %r14
 ; CHECK-NEXT:    pushq %r12
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    subq $3024, %rsp # imm = 0xBD0
 ; CHECK-NEXT:    movl %edi, %ebx
 ; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
 ; CHECK-NEXT:    vmovdqu64 %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $8, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw $8, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movl $buf, %r14d
 ; CHECK-NEXT:    movl $32, %r15d
 ; CHECK-NEXT:    movw $8, %bp
 ; CHECK-NEXT:    movl $buf+2048, %r12d
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    callq foo
-; CHECK-NEXT:    movb $1, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movb $8, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movw $8, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    ldtilecfg {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    testl %ebx, %ebx
 ; CHECK-NEXT:    jle .LBB3_3
 ; CHECK-NEXT:  # %bb.2: # in Loop: Header=BB3_1 Depth=1
 ; CHECK-NEXT:    tileloadd (%r14,%r15), %tmm0
 ; CHECK-NEXT:    movabsq $64, %rax
 ; CHECK-NEXT:    tilestored %tmm0, 1024(%rsp,%rax) # 1024-byte Folded Spill
 ; CHECK-NEXT:    callq foo

llvm/test/CodeGen/X86/AMX/amx-config.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -mattr=+avx512f -verify-machineinstrs | FileCheck %s --check-prefix=AVX512
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -mattr=+avx2 -verify-machineinstrs | FileCheck %s --check-prefix=AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -verify-machineinstrs | FileCheck %s --check-prefix=SSE2
 
 @buf = dso_local global [1024 x i8] zeroinitializer, align 64
 @buf2 = dso_local global [1024 x i8] zeroinitializer, align 64
 
 ; Function Attrs: nounwind uwtable
 define dso_local void @test_api(i32 %0, i16 signext %1, i16 signext %2) {
 ; AVX512-LABEL: test_api:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxord %zmm0, %zmm0, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
 ; AVX512-NEXT:    testl %edi, %edi
 ; AVX512-NEXT:    movsbl %sil, %eax
-; AVX512-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
 ; AVX512-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
 ; AVX512-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
 ; AVX512-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
 ; AVX512-NEXT:    ldtilecfg -{{[0-9]+}}(%rsp)
 ; AVX512-NEXT:    je .LBB0_2
 ; AVX512-NEXT:  # %bb.1:
 ; AVX512-NEXT:    movl $buf, %ecx
 ; AVX512-NEXT:    jmp .LBB0_3
 ; AVX512-NEXT:  .LBB0_2:
 ; AVX512-NEXT:    movl $buf2, %ecx
 ; AVX512-NEXT:  .LBB0_3:

 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX2-LABEL: test_api:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vxorps %ymm0, %ymm0, %ymm0
 ; AVX2-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    testl %edi, %edi
 ; AVX2-NEXT:    movsbl %sil, %eax
-; AVX2-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    ldtilecfg -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    je .LBB0_2
 ; AVX2-NEXT:  # %bb.1:
 ; AVX2-NEXT:    movl $buf, %ecx
 ; AVX2-NEXT:    jmp .LBB0_3
 ; AVX2-NEXT:  .LBB0_2:
 ; AVX2-NEXT:    movl $buf2, %ecx
 ; AVX2-NEXT:  .LBB0_3:

 ;
 ; SSE2-LABEL: test_api:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    movups %xmm0, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movups %xmm0, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movups %xmm0, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movups %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    testl %edi, %edi
 ; SSE2-NEXT:    movsbl %sil, %eax
-; SSE2-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    ldtilecfg -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    je .LBB0_2
 ; SSE2-NEXT:  # %bb.1:
 ; SSE2-NEXT:    movl $buf, %ecx
 ; SSE2-NEXT:    jmp .LBB0_3
 ; SSE2-NEXT:  .LBB0_2:
 ; SSE2-NEXT:    movl $buf2, %ecx
 ; SSE2-NEXT:  .LBB0_3:

llvm/test/CodeGen/X86/AMX/amx-ldtilecfg-insert.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -mattr=+avx512f -verify-machineinstrs | FileCheck %s
 @buf = dso_local global [3072 x i8] zeroinitializer, align 16
 
 define dso_local void @test1(i16 signext %0, i16 signext %1) nounwind {
 ; CHECK-LABEL: test1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
 ; CHECK-NEXT:    vmovdqu64 %zmm0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw $8, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb $8, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    ldtilecfg -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movl $buf, %eax
 ; CHECK-NEXT:    movl $32, %ecx
 ; CHECK-NEXT:    movw $8, %dx
 ; CHECK-NEXT:    tileloadd (%rax,%rcx), %tmm0
 ; CHECK-NEXT:    movl $buf+1024, %eax
 ; CHECK-NEXT:    tileloadd (%rax,%rcx), %tmm1
 ; CHECK-NEXT:    movl $buf+2048, %eax

 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pushq %rbp
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    subq $72, %rsp
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
 ; CHECK-NEXT:    vmovdqu64 %zmm0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    callq foo
 ; CHECK-NEXT:    movb $1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw $8, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $8, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw $8, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb $8, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %bx, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %bx, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movw $8, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movb $8, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %bx, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movw $8, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq foo
 ; CHECK-NEXT:    ldtilecfg {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne .LBB1_3
 ; CHECK-NEXT:  # %bb.1: # %if.true
 ; CHECK-NEXT:    movw $8, %ax
 ; CHECK-NEXT:    tilezero %tmm0
 ; CHECK-NEXT:    movl $32, %ecx
exit:
   ret void
 }
 
 define dso_local void @test3(i16 signext %0, i16 signext %1) nounwind {
 ; CHECK-LABEL: test3:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
 ; CHECK-NEXT:    vmovdqu64 %zmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne .LBB2_2
 ; CHECK-NEXT:  # %bb.1: # %if.true
 ; CHECK-NEXT:    incl %edi
 ; CHECK-NEXT:    jmp .LBB2_3
 ; CHECK-NEXT:  .LBB2_2: # %if.false
 ; CHECK-NEXT:    decl %edi
 ; CHECK-NEXT:  .LBB2_3: # %exit
-; CHECK-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    ldtilecfg -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    tilezero %tmm0
 ; CHECK-NEXT:    movl $buf, %eax
 ; CHECK-NEXT:    movl $32, %ecx
 ; CHECK-NEXT:    tilestored %tmm0, (%rax,%rcx)
 ; CHECK-NEXT:    tilerelease
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq

 
 exit:
   %5 = phi i16 [ %3, %if.true ], [ %4, %if.false ]
   %6 = tail call x86_amx @llvm.x86.tilezero.internal(i16 %5, i16 %1)
   tail call void @llvm.x86.tilestored64.internal(i16 %5, i16 %1, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 0), i64 32, x86_amx %6)
   ret void
 }
 
-; TODO: There's PRA Tile Register Configure bug needs to fix later.
 define dso_local void @test4(i16 signext %0, i16 signext %1) nounwind {
 ; CHECK-LABEL: test4:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
 ; CHECK-NEXT:    vmovdqu64 %zmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne .LBB3_3
 ; CHECK-NEXT:  # %bb.1: # %if.true
 ; CHECK-NEXT:    incl %edi
+; CHECK-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    ldtilecfg -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne .LBB3_4
 ; CHECK-NEXT:  .LBB3_2: # %amx2
-; CHECK-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    ldtilecfg -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movl $32, %eax
 ; CHECK-NEXT:    movl $buf+1024, %ecx
 ; CHECK-NEXT:    tileloadd (%rcx,%rax), %tmm0
 ; CHECK-NEXT:    movl $buf, %ecx
 ; CHECK-NEXT:    tilestored %tmm0, (%rcx,%rax)
 ; CHECK-NEXT:    tilerelease
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:  .LBB3_3: # %if.false
 ; CHECK-NEXT:    decl %edi
+; CHECK-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    ldtilecfg -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne .LBB3_2
 ; CHECK-NEXT:  .LBB3_4: # %amx1
-; CHECK-NEXT:    ldtilecfg -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    tilezero %tmm0
 ; CHECK-NEXT:    movl $buf, %eax
 ; CHECK-NEXT:    movl $32, %ecx
 ; CHECK-NEXT:    tilestored %tmm0, (%rax,%rcx)
 ; CHECK-NEXT:    tilerelease
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   br i1 undef, label %if.true, label %if.false

 }
 
 define dso_local void @test5(i16 signext %0, i16 signext %1) nounwind {
 ; CHECK-LABEL: test5:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    # kill: def $esi killed $esi def $rsi
 ; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
 ; CHECK-NEXT:    vmovdqu64 %zmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    movl $buf, %r8d
 ; CHECK-NEXT:    movl $32, %edx
 ; CHECK-NEXT:    leal -1(%rsi), %ecx
 ; CHECK-NEXT:    jmp .LBB4_1
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB4_3: # %if.false
 ; CHECK-NEXT:    # in Loop: Header=BB4_1 Depth=1
 ; CHECK-NEXT:    movl %ecx, %esi
+; CHECK-NEXT:    movw %cx, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    cmpw $7, %si
 ; CHECK-NEXT:    jne .LBB4_5
 ; CHECK-NEXT:  .LBB4_1: # %loop.bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    ldtilecfg -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne .LBB4_3
 ; CHECK-NEXT:  # %bb.2: # %if.true
 ; CHECK-NEXT:    # in Loop: Header=BB4_1 Depth=1
 ; CHECK-NEXT:    tilezero %tmm0
 ; CHECK-NEXT:    tilestored %tmm0, (%r8,%rdx)
 ; CHECK-NEXT:    cmpw $7, %si

 }
 
 define dso_local void @test6(i16 signext %0) nounwind {
 ; CHECK-LABEL: test6:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
 ; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
 ; CHECK-NEXT:    vmovdqu64 %zmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    xorl %r8d, %r8d
 ; CHECK-NEXT:    movl $buf, %ecx
 ; CHECK-NEXT:    movl $32, %edx
 ; CHECK-NEXT:    xorl %esi, %esi
 ; CHECK-NEXT:    jmp .LBB5_1
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB5_3: # %if.false
 ; CHECK-NEXT:    # in Loop: Header=BB5_1 Depth=1
 ; CHECK-NEXT:    decl %esi
 ; CHECK-NEXT:  .LBB5_4: # %loop.bb2
 ; CHECK-NEXT:    # in Loop: Header=BB5_1 Depth=1
 ; CHECK-NEXT:    leal (%rdi,%rsi), %eax
-; CHECK-NEXT:    cmpw $7, %si
-; CHECK-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    cmpw $7, %si
 ; CHECK-NEXT:    ldtilecfg -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    tilezero %tmm0
 ; CHECK-NEXT:    tilestored %tmm0, (%rcx,%rdx)
 ; CHECK-NEXT:    jne .LBB5_5
 ; CHECK-NEXT:  .LBB5_1: # %loop.bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb %r8b, %r8b
 ; CHECK-NEXT:    jne .LBB5_3

llvm/test/CodeGen/X86/AMX/amx-spill-merge.ll

 ; CHECK-NEXT:    pushq %r14
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    subq $4056, %rsp # imm = 0xFD8
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
 ; CHECK-NEXT:    vmovdqu64 %zmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb $1, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw $8, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb $8, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movw %bx, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw $8, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb $8, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw $8, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %bx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %bx, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movw $8, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %bx, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %bx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    ldtilecfg {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movl $32, %r14d
 ; CHECK-NEXT:    movl $buf+2048, %r15d
 ; CHECK-NEXT:    tileloadd (%r15,%r14), %tmm5
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %if.true

llvm/test/CodeGen/X86/AMX/amx-spill.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -mattr=+avx512f -verify-machineinstrs | FileCheck %s
 
 @buf = dso_local global [1024 x i8] zeroinitializer, align 64
 @buf2 = dso_local global [1024 x i8] zeroinitializer, align 64
 
 define dso_local void @test_api(i32 %0, i16 signext %1, i16 signext %2) nounwind {
 ; CHECK-LABEL: test_api:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    subq $2936, %rsp # imm = 0xB78
 ; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
 ; CHECK-NEXT:    vmovdqu64 %zmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb $1, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movb %dl, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movb %dl, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb %sil, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %si, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb %sil, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb %dl, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb %dl, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb %sil, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movw %si, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %dl, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb %sil, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %dl, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    ldtilecfg {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movl $buf, %r8d
 ; CHECK-NEXT:    movl $32, %eax
 ; CHECK-NEXT:    tileloadd (%r8,%rax), %tmm1
 ; CHECK-NEXT:    tileloadd (%r8,%rax), %tmm1
 ; CHECK-NEXT:    movabsq $64, %rcx
 ; CHECK-NEXT:    tilestored %tmm1, 896(%rsp,%rcx) # 1024-byte Folded Spill
 ; CHECK-NEXT:    tileloadd (%r8,%rax), %tmm3