[AMDGPU] All GWS instructions need aligned VGPR on gfx90a

[rampitec]

Summary

Fixes: SWDEV-288006

[rampitec]

This is alternative to D103197, even though I like D103197 better.

[rampitec]

Fix tidy warnings.

[arsenm]

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

4678–4692

Why can't you use the same hack as is used for AGPR returns?

[rampitec]

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

4678–4692

How does it help here? The RC is selected correctly, but then operand folding substritutes odd subreg. For that reason I have to handlle it in the isOperandLegal() (here). Remove this code and test gws_init_odd_agpr will fail.

llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp

case Intrinsic::amdgcn_ds_ordered_swap: {
       return;
     }
     case Intrinsic::amdgcn_ds_gws_init:
     case Intrinsic::amdgcn_ds_gws_barrier:
     case Intrinsic::amdgcn_ds_gws_sema_br: {
       // Only the first lane is executes, so readfirstlane is safe.
       substituteSimpleCopyRegs(OpdMapper, 1);
       constrainOpWithReadfirstlane(MI, MRI, 2); // M0
+      if (Subtarget.needsAlignedVGPRs()) {
+        Register DataReg = MI.getOperand(1).getReg();
+        constrainGenericRegister(DataReg, AMDGPU::VReg_32_Align2RegClass, MRI);
+      }
       return;
     }
     case Intrinsic::amdgcn_ds_gws_sema_v:
     case Intrinsic::amdgcn_ds_gws_sema_p:
     case Intrinsic::amdgcn_ds_gws_sema_release_all: {
       // Only the first lane is executes, so readfirstlane is safe.
       constrainOpWithReadfirstlane(MI, MRI, 1); // M0
       return;

llvm/lib/Target/AMDGPU/DSInstructions.td

 
 class DS_GWS_1D <string opName>
 : DS_GWS<opName,
   (ins getLdStRegisterOperand<VGPR_32>.ret:$data0, offset:$offset),
   " $data0$offset gds"> {
 
   let has_gws_data0 = 1;
   let hasSideEffects = 1;
+  let hasPostISelHook = 1;
 }
 
 class DS_VOID <string opName> : DS_Pseudo<opName,
   (outs), (ins), ""> {
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 1;
   let UseNamedOperandTable = 0;

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

if ((Node->hasNUsesOfValue(1, 0) &&
       BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
               TII->get(AMDGPU::IMPLICIT_DEF), Def);
     }
     return;
   }
 
   if (TII->isMIMG(MI) && !MI.mayStore())
     AddIMGInit(MI);
+
+  if (Subtarget->needsAlignedVGPRs()) {
+    switch (MI.getOpcode()) {
+    default:
+      break;
+    case AMDGPU::DS_GWS_INIT:
+    case AMDGPU::DS_GWS_SEMA_BR:
+    case AMDGPU::DS_GWS_BARRIER: {
+      const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
+      Register Reg = TII->getNamedOperand(MI, AMDGPU::OpName::data0)->getReg();
+      MRI.setRegClass(Reg, TRI->isAGPR(MRI, Reg)
+                               ? &AMDGPU::AReg_32_Align2RegClass
+                               : &AMDGPU::VReg_32_Align2RegClass);
+      break;
+    }
+    }
+  }
 }
 
 static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL,
                               uint64_t Val) {
   SDValue K = DAG.getTargetConstant(Val, DL, MVT::i32);
   return SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, K), 0);
 }
 

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
       }
     }
   }
 
   if (MO->isReg()) {
     assert(DefinedRC);
     if (!isLegalRegOperand(MRI, OpInfo, *MO))
       return false;
-    bool IsAGPR = RI.isAGPR(MRI, MO->getReg());
+    const MachineOperand &CurOp = MI.getOperand(OpIdx);
+    const auto *RC = RI.getRegClassForReg(MRI, MO->getReg());
+    if (ST.needsAlignedVGPRs() && CurOp.isReg() && &CurOp != MO) {
+      const auto *CurRC = RI.getRegClassForReg(MRI, CurOp.getReg());
+      if (RI.isAlignedRC(CurRC) && RI.hasVectorRegisters(RC)) {
+        unsigned Sub = MO->getSubReg();
+        if (Sub &&
+            !RI.getCompatibleSubRegClass(RI.getEquivalentVGPRClass(RC),
+                                         RI.getEquivalentVGPRClass(CurRC), Sub))
+          return false;
+        if (!Sub && !RI.isAlignedRC(RC))
+          return false;
+      }
+    }
+    bool IsAGPR = RI.hasAGPRs(RC);
     if (IsAGPR && !ST.hasMAIInsts())
       return false;
     unsigned Opc = MI.getOpcode();
     if (IsAGPR &&
         (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
         (MI.mayLoad() || MI.mayStore() || isDS(Opc) || isMIMG(Opc)))
       return false;
     // Atomics should have both vdst and vdata either vgpr or agpr.

llvm/lib/Target/AMDGPU/SIRegisterInfo.h

public:
   const TargetRegisterClass *getVGPRClassForBitWidth(unsigned BitWidth) const;
 
   LLVM_READONLY
   const TargetRegisterClass *getAGPRClassForBitWidth(unsigned BitWidth) const;
 
   LLVM_READONLY
   static const TargetRegisterClass *getSGPRClassForBitWidth(unsigned BitWidth);
 
+  LLVM_READONLY
+  bool isAlignedRC(const TargetRegisterClass *RC) const;
+
   /// Return the 'base' register class for this register.
   /// e.g. SGPR0 => SReg_32, VGPR => VGPR_32 SGPR0_SGPR1 -> SReg_32, etc.
   const TargetRegisterClass *getPhysRegClass(MCRegister Reg) const;
 
   /// \returns true if this class contains only SGPR registers
   bool isSGPRClass(const TargetRegisterClass *RC) const {
     return !hasVGPRs(RC) && !hasAGPRs(RC);
   }

llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp

getAnyVGPRClassForBitWidth(unsigned BitWidth) {
   if (BitWidth <= 1024)
     return &AMDGPU::VReg_1024RegClass;
 
   return nullptr;
 }
 
 static const TargetRegisterClass *
 getAlignedVGPRClassForBitWidth(unsigned BitWidth) {
+  if (BitWidth <= 32)
+    return &AMDGPU::VReg_32_Align2RegClass;
   if (BitWidth <= 64)
     return &AMDGPU::VReg_64_Align2RegClass;
   if (BitWidth <= 96)
     return &AMDGPU::VReg_96_Align2RegClass;
   if (BitWidth <= 128)
     return &AMDGPU::VReg_128_Align2RegClass;
   if (BitWidth <= 160)
     return &AMDGPU::VReg_160_Align2RegClass;
getAnyAGPRClassForBitWidth(unsigned BitWidth) {
   if (BitWidth <= 1024)
     return &AMDGPU::AReg_1024RegClass;
 
   return nullptr;
 }
 
 static const TargetRegisterClass *
 getAlignedAGPRClassForBitWidth(unsigned BitWidth) {
+  if (BitWidth <= 32)
+    return &AMDGPU::AReg_32_Align2RegClass;
   if (BitWidth <= 64)
     return &AMDGPU::AReg_64_Align2RegClass;
   if (BitWidth <= 96)
     return &AMDGPU::AReg_96_Align2RegClass;
   if (BitWidth <= 128)
     return &AMDGPU::AReg_128_Align2RegClass;
   if (BitWidth <= 160)
     return &AMDGPU::AReg_160_Align2RegClass;
SIRegisterInfo::getSGPRClassForBitWidth(unsigned BitWidth) {
   if (BitWidth <= 512)
     return &AMDGPU::SGPR_512RegClass;
   if (BitWidth <= 1024)
     return &AMDGPU::SGPR_1024RegClass;
 
   return nullptr;
 }
 
+bool SIRegisterInfo::isAlignedRC(const TargetRegisterClass *RC) const {
+  if (!RC)
+    return false;
+
+  switch (RC->getID()) {
+  default:
+    return false;
+  case AMDGPU::VReg_32_Align2RegClassID:
+  case AMDGPU::VReg_64_Align2RegClassID:
+  case AMDGPU::VReg_96_Align2RegClassID:
+  case AMDGPU::VReg_128_Align2RegClassID:
+  case AMDGPU::VReg_160_Align2RegClassID:
+  case AMDGPU::VReg_192_Align2RegClassID:
+  case AMDGPU::VReg_256_Align2RegClassID:
+  case AMDGPU::VReg_512_Align2RegClassID:
+  case AMDGPU::VReg_1024_Align2RegClassID:
+  case AMDGPU::AReg_32_Align2RegClassID:
+  case AMDGPU::AReg_64_Align2RegClassID:
+  case AMDGPU::AReg_96_Align2RegClassID:
+  case AMDGPU::AReg_128_Align2RegClassID:
+  case AMDGPU::AReg_160_Align2RegClassID:
+  case AMDGPU::AReg_192_Align2RegClassID:
+  case AMDGPU::AReg_256_Align2RegClassID:
+  case AMDGPU::AReg_512_Align2RegClassID:
+  case AMDGPU::AReg_1024_Align2RegClassID:
+    return true;
+  }
+}
+
 // FIXME: This is very slow. It might be worth creating a map from physreg to
 // register class.
 const TargetRegisterClass *
 SIRegisterInfo::getPhysRegClass(MCRegister Reg) const {
   static const TargetRegisterClass *const BaseClasses[] = {
     &AMDGPU::VGPR_LO16RegClass,
     &AMDGPU::VGPR_HI16RegClass,
     &AMDGPU::SReg_LO16RegClass,
     &AMDGPU::AGPR_LO16RegClass,
     &AMDGPU::VGPR_32RegClass,
     &AMDGPU::SReg_32RegClass,
     &AMDGPU::AGPR_32RegClass,
     &AMDGPU::AGPR_32RegClass,
+    &AMDGPU::VReg_32_Align2RegClass,
+    &AMDGPU::AReg_32_Align2RegClass,
     &AMDGPU::VReg_64_Align2RegClass,
     &AMDGPU::VReg_64RegClass,
     &AMDGPU::SReg_64RegClass,
     &AMDGPU::AReg_64_Align2RegClass,
     &AMDGPU::AReg_64RegClass,
     &AMDGPU::VReg_96_Align2RegClass,
     &AMDGPU::VReg_96RegClass,
     &AMDGPU::SReg_96RegClass,
if (!ARC) {
     return false;
   }
   return getCommonSubClass(ARC, RC) != nullptr;
 }
 
 const TargetRegisterClass *
 SIRegisterInfo::getEquivalentVGPRClass(const TargetRegisterClass *SRC) const {
   unsigned Size = getRegSizeInBits(*SRC);
-  const TargetRegisterClass *VRC = getVGPRClassForBitWidth(Size);
+  const TargetRegisterClass *VRC = isAlignedRC(SRC)
+                                       ? getAlignedVGPRClassForBitWidth(Size)
+                                       : getVGPRClassForBitWidth(Size);
   assert(VRC && "Invalid register class size");
   return VRC;
 }
 
 const TargetRegisterClass *
 SIRegisterInfo::getEquivalentAGPRClass(const TargetRegisterClass *SRC) const {
   unsigned Size = getRegSizeInBits(*SRC);
-  const TargetRegisterClass *ARC = getAGPRClassForBitWidth(Size);
+  const TargetRegisterClass *ARC = isAlignedRC(SRC)
+                                       ? getAlignedAGPRClassForBitWidth(Size)
+                                       : getAGPRClassForBitWidth(Size);
   assert(ARC && "Invalid register class size");
   return ARC;
 }
 
 const TargetRegisterClass *
 SIRegisterInfo::getEquivalentSGPRClass(const TargetRegisterClass *VRC) const {
   unsigned Size = getRegSizeInBits(*VRC);
   if (Size == 32)

llvm/lib/Target/AMDGPU/SIRegisterInfo.td

 multiclass VRegClass<int numRegs, list<ValueType> regTypes, dag regList> {
   // Define the regular class.
   def "" : VRegClassBase<numRegs, regTypes, regList>;
 
   // Define 2-aligned variant
   def _Align2 : VRegClassBase<numRegs, regTypes, (decimate regList, 2)>;
 }
 
+def VReg_32_Align2 : VRegClassBase<1, [OtherVT], (decimate VGPR_32, 2)>;
+
 defm VReg_64 : VRegClass<2, [i64, f64, v2i32, v2f32, v4f16, v4i16, p0, p1, p4],
                                 (add VGPR_64)>;
 defm VReg_96 : VRegClass<3, [v3i32, v3f32], (add VGPR_96)>;
 defm VReg_128 : VRegClass<4, [v4i32, v4f32, v2i64, v2f64], (add VGPR_128)>;
 defm VReg_160 : VRegClass<5, [v5i32, v5f32], (add VGPR_160)>;
 
 defm VReg_192 : VRegClass<6, [untyped], (add VGPR_192)>;
 defm VReg_256 : VRegClass<8, [v8i32, v8f32, v4i64, v4f64], (add VGPR_256)>;
 defm VReg_512 : VRegClass<16, [v16i32, v16f32, v8i64, v8f64], (add VGPR_512)>;
 defm VReg_1024 : VRegClass<32, [v32i32, v32f32, v16i64, v16f64], (add VGPR_1024)>;
 
 multiclass ARegClass<int numRegs, list<ValueType> regTypes, dag regList> {
   let CopyCost = !add(numRegs, numRegs, 1) in {
     // Define the regular class.
     def "" : VRegClassBase<numRegs, regTypes, regList>;
 
     // Define 2-aligned variant
     def _Align2 : VRegClassBase<numRegs, regTypes, (decimate regList, 2)>;
   }
 }
 
+def AReg_32_Align2 : VRegClassBase<1, [OtherVT], (decimate AGPR_32, 2)>;
+
 defm AReg_64 : ARegClass<2, [i64, f64, v2i32, v2f32, v4f16, v4i16],
                         (add AGPR_64)>;
 defm AReg_96 : ARegClass<3, [v3i32, v3f32], (add AGPR_96)>;
 defm AReg_128 : ARegClass<4, [v4i32, v4f32, v2i64, v2f64], (add AGPR_128)>;
 defm AReg_160 : ARegClass<5, [v5i32, v5f32], (add AGPR_160)>;
 defm AReg_192 : ARegClass<6, [untyped], (add AGPR_192)>;
 defm AReg_256 : ARegClass<8, [v8i32, v8f32, v4i64, v4f64], (add AGPR_256)>;
 defm AReg_512 : ARegClass<16, [v16i32, v16f32, v8i64, v8f64], (add AGPR_512)>;

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

unsigned getRegBitWidth(unsigned RCID) {
   case AMDGPU::VGPR_32RegClassID:
   case AMDGPU::VRegOrLds_32RegClassID:
   case AMDGPU::AGPR_32RegClassID:
   case AMDGPU::VS_32RegClassID:
   case AMDGPU::AV_32RegClassID:
   case AMDGPU::SReg_32RegClassID:
   case AMDGPU::SReg_32_XM0RegClassID:
   case AMDGPU::SRegOrLds_32RegClassID:
+  case AMDGPU::VReg_32_Align2RegClassID:
+  case AMDGPU::AReg_32_Align2RegClassID:
     return 32;
   case AMDGPU::SGPR_64RegClassID:
   case AMDGPU::VS_64RegClassID:
   case AMDGPU::AV_64RegClassID:
   case AMDGPU::SReg_64RegClassID:
   case AMDGPU::VReg_64RegClassID:
   case AMDGPU::AReg_64RegClassID:
   case AMDGPU::SReg_64_XEXECRegClassID:

llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll

 }
 
 ; Check support for returning several floats
 define double @test_multiple_register_outputs_mixed() #0 {
   ; CHECK-LABEL: name: test_multiple_register_outputs_mixed
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
-  ; CHECK:   INLINEASM &"v_mov_b32 $0, 0; v_add_f64 $1, 0, 0", 0 /* attdialect */, 1835018 /* regdef:VGPR_32 */, def %1, 2883594 /* regdef:VReg_64 */, def %2
+  ; CHECK:   INLINEASM &"v_mov_b32 $0, 0; v_add_f64 $1, 0, 0", 0 /* attdialect */, 1835018 /* regdef:VGPR_32 */, def %1, 3014666 /* regdef:VReg_64 */, def %2
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY %1
   ; CHECK:   [[COPY2:%[0-9]+]]:_(s64) = COPY %2
   ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](s64)
   ; CHECK:   $vgpr0 = COPY [[UV]](s32)
   ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
   ; CHECK:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
   ; CHECK:   S_SETPC_B64_return [[COPY3]], implicit $vgpr0, implicit $vgpr1
   %1 = call { float, double } asm "v_mov_b32 $0, 0; v_add_f64 $1, 0, 0", "=v,=v"()

llvm/test/CodeGen/AMDGPU/ds_gws_align.ll

+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx908 -o - -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX908 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx90a -o - -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX90A %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx908 -o - -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX908 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx90a -o - -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX90A %s
+
+; GCN-LABEL: {{^}}gws_init_odd_reg:
+; GFX908-DAG: ds_gws_init v1 gds
+; GFX90A-DAG: ds_gws_init v2 gds
+; GCN-DAG:    ds_gws_init v0 gds
+define amdgpu_ps void @gws_init_odd_reg(<2 x i32> %arg) {
+  %vgpr.0 = extractelement <2 x i32> %arg, i32 0
+  %vgpr.1 = extractelement <2 x i32> %arg, i32 1
+  call void @llvm.amdgcn.ds.gws.init(i32 %vgpr.0, i32 0)
+  call void @llvm.amdgcn.ds.gws.init(i32 %vgpr.1, i32 0)
+  ret void
+}
+
+; GCN-LABEL: {{^}}gws_sema_br_odd_reg:
+; GFX908-DAG: ds_gws_sema_br v1 gds
+; GFX90A-DAG: ds_gws_sema_br v2 gds
+; GCN-DAG:    ds_gws_sema_br v0 gds
+define amdgpu_ps void @gws_sema_br_odd_reg(<2 x i32> %arg) {
+  %vgpr.0 = extractelement <2 x i32> %arg, i32 0
+  %vgpr.1 = extractelement <2 x i32> %arg, i32 1
+  call void @llvm.amdgcn.ds.gws.sema.br(i32 %vgpr.0, i32 0)
+  call void @llvm.amdgcn.ds.gws.sema.br(i32 %vgpr.1, i32 0)
+  ret void
+}
+
+; GCN-LABEL: {{^}}gws_barrier_odd_reg:
+; GFX908-DAG: ds_gws_barrier v1 gds
+; GFX90A-DAG: ds_gws_barrier v2 gds
+; GCN-DAG:    ds_gws_barrier v0 gds
+define amdgpu_ps void @gws_barrier_odd_reg(<2 x i32> %arg) {
+  %vgpr.0 = extractelement <2 x i32> %arg, i32 0
+  %vgpr.1 = extractelement <2 x i32> %arg, i32 1
+  call void @llvm.amdgcn.ds.gws.barrier(i32 %vgpr.0, i32 0)
+  call void @llvm.amdgcn.ds.gws.barrier(i32 %vgpr.1, i32 0)
+  ret void
+}
+
+; GCN-LABEL: {{^}}gws_init_odd_agpr:
+; GFX908-COUNT-2: ds_gws_init v{{[0-9]+}} gds
+; GFX90A:         ds_gws_init a{{[0-9]*[02468]}} gds
+; GFX90A:         ds_gws_init v{{[0-9]*[02468]}} gds
+define amdgpu_ps void @gws_init_odd_agpr(<4 x i32> %arg) {
+bb:
+  %mai = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 1, i32 2, <4 x i32> %arg, i32 0, i32 0, i32 0)
+  %agpr.0 = extractelement <4 x i32> %mai, i32 0
+  %agpr.1 = extractelement <4 x i32> %mai, i32 1
+  call void @llvm.amdgcn.ds.gws.init(i32 %agpr.0, i32 0)
+  call void @llvm.amdgcn.ds.gws.init(i32 %agpr.1, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.ds.gws.init(i32, i32)
+declare void @llvm.amdgcn.ds.gws.sema.br(i32, i32)
+declare void @llvm.amdgcn.ds.gws.barrier(i32, i32)
+declare <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32, i32, <4 x i32>, i32, i32, i32)

llvm/test/CodeGen/AMDGPU/inline-asm.i128.ll

 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -stop-after=finalize-isel -o - %s | FileCheck -check-prefix=GFX908 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -stop-after=finalize-isel -o - %s | FileCheck -check-prefix=GFX90A %s
 
 ; Make sure we only use one 128-bit register instead of 2 for i128 asm
 ; constraints
 
 define amdgpu_kernel void @s_input_output_i128() {
   ; GFX908-LABEL: name: s_input_output_i128
   ; GFX908: bb.0 (%ir-block.0):
-  ; GFX908:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 4784138 /* regdef:SGPR_128 */, def %4
+  ; GFX908:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 5046282 /* regdef:SGPR_128 */, def %4
   ; GFX908:   [[COPY:%[0-9]+]]:sgpr_128 = COPY %4
-  ; GFX908:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 4784137 /* reguse:SGPR_128 */, [[COPY]]
+  ; GFX908:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 5046281 /* reguse:SGPR_128 */, [[COPY]]
   ; GFX908:   S_ENDPGM 0
   ; GFX90A-LABEL: name: s_input_output_i128
   ; GFX90A: bb.0 (%ir-block.0):
-  ; GFX90A:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 4784138 /* regdef:SGPR_128 */, def %4
+  ; GFX90A:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 5046282 /* regdef:SGPR_128 */, def %4
   ; GFX90A:   [[COPY:%[0-9]+]]:sgpr_128 = COPY %4
-  ; GFX90A:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 4784137 /* reguse:SGPR_128 */, [[COPY]]
+  ; GFX90A:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 5046281 /* reguse:SGPR_128 */, [[COPY]]
   ; GFX90A:   S_ENDPGM 0
   %val = tail call i128 asm sideeffect "; def $0", "=s"()
   call void asm sideeffect "; use $0", "s"(i128 %val)
   ret void
 }
 
 define amdgpu_kernel void @v_input_output_i128() {
   ; GFX908-LABEL: name: v_input_output_i128
   ; GFX908: bb.0 (%ir-block.0):
-  ; GFX908:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 4390922 /* regdef:VReg_128 */, def %4
+  ; GFX908:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 4653066 /* regdef:VReg_128 */, def %4
   ; GFX908:   [[COPY:%[0-9]+]]:vreg_128 = COPY %4
-  ; GFX908:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 4390921 /* reguse:VReg_128 */, [[COPY]]
+  ; GFX908:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 4653065 /* reguse:VReg_128 */, [[COPY]]
   ; GFX908:   S_ENDPGM 0
   ; GFX90A-LABEL: name: v_input_output_i128
   ; GFX90A: bb.0 (%ir-block.0):
-  ; GFX90A:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 4521994 /* regdef:VReg_128_Align2 */, def %4
+  ; GFX90A:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 4784138 /* regdef:VReg_128_Align2 */, def %4
   ; GFX90A:   [[COPY:%[0-9]+]]:vreg_128_align2 = COPY %4
-  ; GFX90A:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 4521993 /* reguse:VReg_128_Align2 */, [[COPY]]
+  ; GFX90A:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 4784137 /* reguse:VReg_128_Align2 */, [[COPY]]
   ; GFX90A:   S_ENDPGM 0
   %val = tail call i128 asm sideeffect "; def $0", "=v"()
   call void asm sideeffect "; use $0", "v"(i128 %val)
   ret void
 }
 
 define amdgpu_kernel void @a_input_output_i128() {
   ; GFX908-LABEL: name: a_input_output_i128
   ; GFX908: bb.0 (%ir-block.0):
-  ; GFX908:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 4325386 /* regdef:AReg_128 */, def %4
+  ; GFX908:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 4587530 /* regdef:AReg_128 */, def %4
   ; GFX908:   [[COPY:%[0-9]+]]:areg_128 = COPY %4
-  ; GFX908:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 4325385 /* reguse:AReg_128 */, [[COPY]]
+  ; GFX908:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 4587529 /* reguse:AReg_128 */, [[COPY]]
   ; GFX908:   S_ENDPGM 0
   ; GFX90A-LABEL: name: a_input_output_i128
   ; GFX90A: bb.0 (%ir-block.0):
-  ; GFX90A:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 4456458 /* regdef:AReg_128_Align2 */, def %4
+  ; GFX90A:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 4718602 /* regdef:AReg_128_Align2 */, def %4
   ; GFX90A:   [[COPY:%[0-9]+]]:areg_128_align2 = COPY %4
-  ; GFX90A:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 4456457 /* reguse:AReg_128_Align2 */, [[COPY]]
+  ; GFX90A:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 4718601 /* reguse:AReg_128_Align2 */, [[COPY]]
   ; GFX90A:   S_ENDPGM 0
   %val = call i128 asm sideeffect "; def $0", "=a"()
   call void asm sideeffect "; use $0", "a"(i128 %val)
   ret void
 }