[AMDGPU] Enforce alignment of image vaddr on gfx90a

[rampitec]

Summary

Even though single address image instructions only use a single VGPR
HW accesses 4 or 5 which creates alignment requirement.

Fixes: SWDEV-316648

[bcahoon]

This looks good. Just a couple of minor comments.

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

8440

This method returns bool, but none of the callers check the result.

llvm/lib/Target/AMDGPU/SIInstrInfo.h

1156

spelling, regardless

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.gfx90a.ll

73

Is this pattern correct? I believe it would allow v21 for example.

[rampitec]

Addressed review comments.

llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

bool AMDGPUInstructionSelector::selectDSGWSIntrinsic(MachineInstr &MI,
 
   // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
   // offset field) % 64. Some versions of the programming guide omit the m0
   // part, or claim it's from offset 0.
   auto MIB = BuildMI(*MBB, &MI, DL, TII.get(gwsIntrinToOpcode(IID)));
 
   if (HasVSrc) {
     Register VSrc = MI.getOperand(1).getReg();
-
-    if (STI.needsAlignedVGPRs()) {
-      // Add implicit aligned super-reg to force alignment on the data operand.
-      Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
-      BuildMI(*MBB, &*MIB, DL, TII.get(AMDGPU::IMPLICIT_DEF), Undef);
-      Register NewVR =
-          MRI->createVirtualRegister(&AMDGPU::VReg_64_Align2RegClass);
-      BuildMI(*MBB, &*MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), NewVR)
-          .addReg(VSrc, 0, MI.getOperand(1).getSubReg())
-          .addImm(AMDGPU::sub0)
-          .addReg(Undef)
-          .addImm(AMDGPU::sub1);
-      MIB.addReg(NewVR, 0, AMDGPU::sub0);
-      MIB.addReg(NewVR, RegState::Implicit);
-    } else {
-      MIB.addReg(VSrc);
-    }
 
     if (!RBI.constrainGenericRegister(VSrc, AMDGPU::VGPR_32RegClass, *MRI))
       return false;
   }
 
   MIB.addImm(ImmOffset)
      .cloneMemRefs(MI);
 
+  TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::data0);
+
   MI.eraseFromParent();
   return true;
 }
 
 bool AMDGPUInstructionSelector::selectDSAppendConsume(MachineInstr &MI,
                                                       bool IsAppend) const {
   Register PtrBase = MI.getOperand(2).getReg();
   LLT PtrTy = MRI->getType(PtrBase);
if (STI.usePRTStrictNull()) {
         RegSeq.addReg(Undef).addImm(Sub);
       RegSeq.addReg(Zero).addImm(Parts.back());
     }
     MIB.addReg(Tied, RegState::Implicit);
     MIB->tieOperands(0, MIB->getNumOperands() - 1);
   }
 
   MI.eraseFromParent();
-  return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
+  constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
+  TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::vaddr);
+  return true;
 }
 
 bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
     MachineInstr &I) const {
   unsigned IntrinsicID = I.getIntrinsicID();
   switch (IntrinsicID) {
   case Intrinsic::amdgcn_end_cf:
     return selectEndCfIntrinsic(I);

llvm/lib/Target/AMDGPU/MIMGInstructions.td

multiclass MIMG_Store_Addr_Helper <mimgopc op, string asm,
                                   RegisterClass data_rc,
                                   bit enableDisasm> {
   let mayLoad = 0, mayStore = 1, hasSideEffects = 0, hasPostISelHook = 0,
       DisableWQM = 1, ssamp = 0 in {
     let VAddrDwords = 1 in {
       if op.HAS_BASE then {
         def _V1 : MIMG_Store_Helper <op, asm, data_rc, VGPR_32,
                                      !if(enableDisasm, "AMDGPU", "")>;
+        let hasPostISelHook = 1 in
         def _V1_gfx90a : MIMG_Store_Helper_gfx90a <op, asm, data_rc, VGPR_32,
                                      !if(enableDisasm, "GFX90A", "")>;
         def _V1_gfx10 : MIMG_Store_gfx10 <op, asm, data_rc, VGPR_32,
                                           !if(enableDisasm, "AMDGPU", "")>;
       }
     }
     let VAddrDwords = 2 in {
       if op.HAS_BASE then {
let hasSideEffects = 1, // FIXME: remove this
       mayLoad = 1, mayStore = 1, hasPostISelHook = 0, DisableWQM = 1,
       ssamp = 0, FPAtomic = isFP in {
     let VAddrDwords = 1 in {
       if op.HAS_SI then {
         def _V1_si : MIMG_Atomic_si <op, asm, data_rc, VGPR_32, enableDasm>;
       }
       if op.HAS_VI then {
         def _V1_vi : MIMG_Atomic_vi <op, asm, data_rc, VGPR_32, enableDasm>;
+        let hasPostISelHook = 1 in
         def _V1_gfx90a : MIMG_Atomic_gfx90a <op, asm, data_rc, VGPR_32, enableDasm>;
       }
       if op.HAS_BASE then {
         def _V1_gfx10 : MIMG_Atomic_gfx10 <op, asm, data_rc, VGPR_32, enableDasm>;
       }
     }
     let VAddrDwords = 2 in {
       if op.HAS_SI then {

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter(
   case AMDGPU::V_SUBBREV_U32_e32:
     // These instructions have an implicit use of vcc which counts towards the
     // constant bus limit.
     TII->legalizeOperands(MI);
     return BB;
   case AMDGPU::DS_GWS_INIT:
   case AMDGPU::DS_GWS_SEMA_BR:
   case AMDGPU::DS_GWS_BARRIER:
-    if (Subtarget->needsAlignedVGPRs()) {
-      // Add implicit aligned super-reg to force alignment on the data operand.
-      const DebugLoc &DL = MI.getDebugLoc();
-      MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
-      const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
-      MachineOperand *Op = TII->getNamedOperand(MI, AMDGPU::OpName::data0);
-      Register DataReg = Op->getReg();
-      bool IsAGPR = TRI->isAGPR(MRI, DataReg);
-      Register Undef = MRI.createVirtualRegister(
-          IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);
-      BuildMI(*BB, MI, DL, TII->get(AMDGPU::IMPLICIT_DEF), Undef);
-      Register NewVR =
-          MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass
-                                           : &AMDGPU::VReg_64_Align2RegClass);
-      BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), NewVR)
-          .addReg(DataReg, 0, Op->getSubReg())
-          .addImm(AMDGPU::sub0)
-          .addReg(Undef)
-          .addImm(AMDGPU::sub1);
-      Op->setReg(NewVR);
-      Op->setSubReg(AMDGPU::sub0);
-      MI.addOperand(MachineOperand::CreateReg(NewVR, false, true));
-    }
+    TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::data0);
     LLVM_FALLTHROUGH;
   case AMDGPU::DS_GWS_SEMA_V:
   case AMDGPU::DS_GWS_SEMA_P:
   case AMDGPU::DS_GWS_SEMA_RELEASE_ALL:
     // A s_waitcnt 0 is required to be the instruction immediately following.
     if (getSubtarget()->hasGWSAutoReplay()) {
       bundleInstWithWaitcnt(MI);
       return BB;
if (MI.getDesc().OpInfo) {
           }
         }
       }
     }
 
     return;
   }
 
-  if (TII->isMIMG(MI) && !MI.mayStore())
+  if (TII->isMIMG(MI)) {
+    if (!MI.mayStore())
       AddIMGInit(MI);
+    TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::vaddr);
+  }
 }
 
 static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL,
                               uint64_t Val) {
   SDValue K = DAG.getTargetConstant(Val, DL, MVT::i32);
   return SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, K), 0);
 }
 

llvm/lib/Target/AMDGPU/SIInstrInfo.h

const MIRFormatter *getMIRFormatter() const override {
     if (!Formatter.get())
       Formatter = std::make_unique<AMDGPUMIRFormatter>();
     return Formatter.get();
   }
 
   static unsigned getDSShaderTypeValue(const MachineFunction &MF);
 
   const TargetSchedModel &getSchedModel() const { return SchedModel; }
+
+  // Enforce operand's \p OpName even alignment if required by target.
+  // This is used if an operand is a 32 bit register but needs to be aligned
+  // regardless.
+  void enforceOperandRCAlignment(MachineInstr &MI, unsigned OpName) const;
 };
 
 /// \brief Returns true if a reg:subreg pair P has a TRC class
 inline bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P,
                          const TargetRegisterClass &TRC,
                          MachineRegisterInfo &MRI) {
   auto *RC = MRI.getRegClass(P.Reg);
   if (!P.SubReg)

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

if (ST.hasGFX90AInsts()) {
           (Data2 && RI.isAGPR(MRI, Data2->getReg()))) {
         ErrInfo = "Invalid register class: "
                   "agpr loads and stores not supported on this GPU";
         return false;
       }
     }
   }
 
-  if (ST.needsAlignedVGPRs() &&
-      (MI.getOpcode() == AMDGPU::DS_GWS_INIT ||
-       MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR ||
-       MI.getOpcode() == AMDGPU::DS_GWS_BARRIER)) {
-    const MachineOperand *Op = getNamedOperand(MI, AMDGPU::OpName::data0);
+  if (ST.needsAlignedVGPRs()) {
+    const auto isAlignedReg = [&MI, &MRI, this](unsigned OpName) -> bool {
+      const MachineOperand *Op = getNamedOperand(MI, OpName);
+      if (!Op)
+        return true;
       Register Reg = Op->getReg();
-    bool Aligned = true;
-    if (Reg.isPhysical()) {
-      Aligned = !(RI.getHWRegIndex(Reg) & 1);
-    } else {
+      if (Reg.isPhysical())
+        return !(RI.getHWRegIndex(Reg) & 1);
       const TargetRegisterClass &RC = *MRI.getRegClass(Reg);
-      Aligned = RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
-                !(RI.getChannelFromSubReg(Op->getSubReg()) & 1);
-    }
+      return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
+    };
 
-    if (!Aligned) {
+    if (MI.getOpcode() == AMDGPU::DS_GWS_INIT ||
+        MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR ||
+        MI.getOpcode() == AMDGPU::DS_GWS_BARRIER) {
+
+      if (!isAlignedReg(AMDGPU::OpName::data0)) {
         ErrInfo = "Subtarget requires even aligned vector registers "
                   "for DS_GWS instructions";
         return false;
       }
     }
 
+    if (isMIMG(MI)) {
+      if (!isAlignedReg(AMDGPU::OpName::vaddr)) {
+        ErrInfo = "Subtarget requires even aligned vector registers "
+                  "for vaddr operand of image instructions";
+        return false;
+      }
+    }
+  }
+
   if (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
       !ST.hasGFX90AInsts()) {
     const MachineOperand *Src = getNamedOperand(MI, AMDGPU::OpName::src0);
     if (Src->isReg() && RI.isSGPRReg(MRI, Src->getReg())) {
       ErrInfo = "Invalid register class: "
                 "v_accvgpr_write with an SGPR is not supported on this GPU";
       return false;
     }
bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
   case AMDGPU::S_CMPK_GT_I32:
     return optimizeCmpAnd(0, 32, false, true);
   case AMDGPU::S_CMP_LG_U64:
     return optimizeCmpAnd(0, 64, true, false);
   }
 
   return false;
 }
+
+void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI,
+                                            unsigned OpName) const {
+  if (!ST.needsAlignedVGPRs())
+    return;
+
+  int OpNo = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName);
+  if (OpNo < 0)
+    return;
+  MachineOperand &Op = MI.getOperand(OpNo);
+  if (getOpSize(MI, OpNo) > 4)
+    return;
+
+  // Add implicit aligned super-reg to force alignment on the data operand.
+  const DebugLoc &DL = MI.getDebugLoc();
+  MachineBasicBlock *BB = MI.getParent();
+  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
+  Register DataReg = Op.getReg();
+  bool IsAGPR = RI.isAGPR(MRI, DataReg);
+  Register Undef = MRI.createVirtualRegister(
+      IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);
+  BuildMI(*BB, MI, DL, get(AMDGPU::IMPLICIT_DEF), Undef);
+  Register NewVR =
+      MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass
+                                       : &AMDGPU::VReg_64_Align2RegClass);
+  BuildMI(*BB, MI, DL, get(AMDGPU::REG_SEQUENCE), NewVR)
+      .addReg(DataReg, 0, Op.getSubReg())
+      .addImm(AMDGPU::sub0)
+      .addReg(Undef)
+      .addImm(AMDGPU::sub1);
+  Op.setReg(NewVR);
+  Op.setSubReg(AMDGPU::sub0);
+  MI.addOperand(MachineOperand::CreateReg(NewVR, false, true));
+}

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - %s | FileCheck -check-prefix=GFX6 %s
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -o - %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - %s | FileCheck -check-prefix=GFX900 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx90a -o - %s | FileCheck -check-prefix=GFX90A %s
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10 %s
 
-
 define amdgpu_ps float @atomic_swap_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 ; GFX6-LABEL: atomic_swap_i32_1d:
 ; GFX6:       ; %bb.0: ; %main_body
 ; GFX6-NEXT:    s_mov_b32 s0, s2
 ; GFX6-NEXT:    s_mov_b32 s1, s3
 ; GFX6-NEXT:    s_mov_b32 s2, s4
 ; GFX6-NEXT:    s_mov_b32 s3, s5
 ; GFX6-NEXT:    s_mov_b32 s4, s6

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_swap v0, v1, s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_swap_i32_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_swap v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_swap_i32_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
+; GFX90A-NEXT:    image_atomic_swap v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_swap_i32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i32_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
+; GFX90A-NEXT:    image_atomic_add v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_sub v0, v1, s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_sub_i32_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_sub v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_sub_i32_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
+; GFX90A-NEXT:    image_atomic_sub v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_sub_i32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_smin v0, v1, s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_smin_i32_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_smin v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_smin_i32_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
+; GFX90A-NEXT:    image_atomic_smin v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_smin_i32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_umin v0, v1, s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_umin_i32_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_umin v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_umin_i32_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
+; GFX90A-NEXT:    image_atomic_umin v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_umin_i32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_smax v0, v1, s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_smax_i32_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_smax v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_smax_i32_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
+; GFX90A-NEXT:    image_atomic_smax v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_smax_i32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_umax v0, v1, s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_umax_i32_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_umax v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_umax_i32_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
+; GFX90A-NEXT:    image_atomic_umax v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_umax_i32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_and v0, v1, s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_and_i32_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_and v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_and_i32_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
+; GFX90A-NEXT:    image_atomic_and v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_and_i32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_or v0, v1, s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_or_i32_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_or v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_or_i32_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
+; GFX90A-NEXT:    image_atomic_or v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_or_i32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_xor v0, v1, s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_xor_i32_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_xor v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_xor_i32_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
+; GFX90A-NEXT:    image_atomic_xor v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_xor_i32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_inc v0, v1, s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_inc_i32_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_inc v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_inc_i32_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
+; GFX90A-NEXT:    image_atomic_inc v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_inc_i32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_dec v0, v1, s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_dec_i32_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_dec v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_dec_i32_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
+; GFX90A-NEXT:    image_atomic_dec v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_dec_i32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_cmpswap_i32_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_cmpswap_i32_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_cmpswap_i32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s3, s5
 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_endpgm
 ;
+; GFX900-LABEL: atomic_cmpswap_i32_1d_no_return:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_endpgm
+;
+; GFX90A-LABEL: atomic_cmpswap_i32_1d_no_return:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_endpgm
+;
 ; GFX10-LABEL: atomic_cmpswap_i32_1d_no_return:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i32_2d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_2d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
+; GFX90A-NEXT:    image_atomic_add v0, v[4:5], s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i32_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i32_3d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_3d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
+; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
+; GFX90A-NEXT:    image_atomic_add v0, v[4:6], s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i32_3d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc da
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i32_cube:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc da
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_cube:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
+; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
+; GFX90A-NEXT:    image_atomic_add v0, v[4:6], s[0:7] dmask:0x1 unorm glc da
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i32_cube:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc da
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i32_1darray:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc da
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_1darray:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
+; GFX90A-NEXT:    image_atomic_add v0, v[4:5], s[0:7] dmask:0x1 unorm glc da
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i32_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc da
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i32_2darray:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc da
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_2darray:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
+; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
+; GFX90A-NEXT:    image_atomic_add v0, v[4:6], s[0:7] dmask:0x1 unorm glc da
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i32_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i32_2dmsaa:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_2dmsaa:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v2
+; GFX90A-NEXT:    v_mov_b32_e32 v6, v3
+; GFX90A-NEXT:    image_atomic_add v0, v[4:6], s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i32_2dmsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v0, v[1:4], s[0:7] dmask:0x1 unorm glc da
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i32_2darraymsaa:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v0, v[1:4], s[0:7] dmask:0x1 unorm glc da
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_2darraymsaa:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v6, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v7, v2
+; GFX90A-NEXT:    v_mov_b32_e32 v8, v3
+; GFX90A-NEXT:    v_mov_b32_e32 v9, v4
+; GFX90A-NEXT:    image_atomic_add v0, v[6:9], s[0:7] dmask:0x1 unorm glc da
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i32_2darraymsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc slc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i32_1d_slc:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc slc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_1d_slc:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v1
+; GFX90A-NEXT:    image_atomic_add v0, v2, s[0:7] dmask:0x1 unorm glc slc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i32_1d_slc:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_swap_i64_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_swap_i64_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_swap_i64_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i64_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i64_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_sub_i64_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_sub_i64_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_sub_i64_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_smin_i64_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_smin_i64_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_smin_i64_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_umin_i64_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_umin_i64_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_umin_i64_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_smax_i64_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_smax_i64_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_smax_i64_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_umax_i64_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_umax_i64_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_umax_i64_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_and_i64_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_and_i64_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_and_i64_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_or_i64_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_or_i64_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_or_i64_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_xor_i64_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_xor_i64_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_xor_i64_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_inc_i64_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_inc_i64_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_inc_i64_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_dec_i64_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_dec_i64_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_dec_i64_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_cmpswap_i64_1d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_cmpswap_i64_1d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_cmpswap_i64_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s3, s5
 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc
 ; GFX8-NEXT:    s_endpgm
 ;
+; GFX900-LABEL: atomic_cmpswap_i64_1d_no_return:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc
+; GFX900-NEXT:    s_endpgm
+;
+; GFX90A-LABEL: atomic_cmpswap_i64_1d_no_return:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc
+; GFX90A-NEXT:    s_endpgm
+;
 ; GFX10-LABEL: atomic_cmpswap_i64_1d_no_return:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i64_2d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_2d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i64_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i64_3d:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_3d:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i64_3d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc da
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i64_cube:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc da
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_cube:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc da
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i64_cube:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc da
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i64_1darray:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc da
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_1darray:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc da
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i64_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc da
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i64_2darray:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc da
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_2darray:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc da
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i64_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i64_2dmsaa:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_2dmsaa:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i64_2dmsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v[0:1], v[2:5], s[0:7] dmask:0x3 unorm glc da
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i64_2darraymsaa:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v[0:1], v[2:5], s[0:7] dmask:0x3 unorm glc da
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_2darraymsaa:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_add v[0:1], v[2:5], s[0:7] dmask:0x3 unorm glc da
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i64_2darraymsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GFX8-NEXT:    s_mov_b32 s4, s6
 ; GFX8-NEXT:    s_mov_b32 s5, s7
 ; GFX8-NEXT:    s_mov_b32 s6, s8
 ; GFX8-NEXT:    s_mov_b32 s7, s9
 ; GFX8-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc slc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
+; GFX900-LABEL: atomic_add_i64_1d_slc:
+; GFX900:       ; %bb.0: ; %main_body
+; GFX900-NEXT:    s_mov_b32 s0, s2
+; GFX900-NEXT:    s_mov_b32 s1, s3
+; GFX900-NEXT:    s_mov_b32 s2, s4
+; GFX900-NEXT:    s_mov_b32 s3, s5
+; GFX900-NEXT:    s_mov_b32 s4, s6
+; GFX900-NEXT:    s_mov_b32 s5, s7
+; GFX900-NEXT:    s_mov_b32 s6, s8
+; GFX900-NEXT:    s_mov_b32 s7, s9
+; GFX900-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc slc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_1d_slc:
+; GFX90A:       ; %bb.0: ; %main_body
+; GFX90A-NEXT:    s_mov_b32 s0, s2
+; GFX90A-NEXT:    s_mov_b32 s1, s3
+; GFX90A-NEXT:    s_mov_b32 s2, s4
+; GFX90A-NEXT:    s_mov_b32 s3, s5
+; GFX90A-NEXT:    s_mov_b32 s4, s6
+; GFX90A-NEXT:    s_mov_b32 s5, s7
+; GFX90A-NEXT:    s_mov_b32 s6, s8
+; GFX90A-NEXT:    s_mov_b32 s7, s9
+; GFX90A-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc slc
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    ; return to shader part epilog
+;
 ; GFX10-LABEL: atomic_add_i64_1d_slc:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s2, s4
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s4, s6
 ; GFX10-NEXT:    s_mov_b32 s5, s7

llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir

liveins:
   - { reg: '$vgpr0_vgpr1' }
   - { reg: '$vgpr2' }
 body:             |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2
 
     %0:vreg_64_align2 = COPY $vgpr0_vgpr1
     %1:vgpr_32 = COPY $vgpr2
-    %2:vreg_160_align2 = IMAGE_LOAD_V5_V1 %1, undef %3:sgpr_256, 0, 0, 0, 0, 0, 1, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 4)
+    %2:vreg_160_align2 = IMAGE_LOAD_V5_V1 %0.sub0, undef %3:sgpr_256, 0, 0, 0, 0, 0, 1, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 4)
     GLOBAL_STORE_DWORDX4 %0, %2.sub0_sub1_sub2_sub3, 0, 0, implicit $exec
     GLOBAL_STORE_DWORD %0, %1, 0, 0, implicit $exec
 ...
 
 ---
 # GCN-LABEL: name: alloc_vgpr_256
 # GFX90A: $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11 = COPY
 name:            alloc_vgpr_256

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.dim.ll

 ; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX6789 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX6789 %s
-; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX90A %s
 ; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=GCN -check-prefix=GFX10 %s
 
 ; GCN-LABEL: {{^}}atomic_swap_1d:
 ; GFX6789: image_atomic_swap v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_swap v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps float @atomic_swap_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_swap_1d_i64:
 ; GFX6789: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 unorm glc{{$}}
+; GFX90A: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 unorm glc{{$}}
 ; GFX10: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps <2 x float> @atomic_swap_1d_i64(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
 main_body:
   %v = call i64 @llvm.amdgcn.image.atomic.swap.1d.i64.i32(i64 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i64 %v to <2 x float>
   ret <2 x float> %out
 }
 
 ; GCN-LABEL: {{^}}atomic_add_1d:
 ; GFX6789: image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_add v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps float @atomic_add_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.add.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_sub_1d:
 ; GFX6789: image_atomic_sub v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_sub v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_sub v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps float @atomic_sub_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.sub.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_smin_1d:
 ; GFX6789: image_atomic_smin v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_smin v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_smin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps float @atomic_smin_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.smin.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_umin_1d:
 ; GFX6789: image_atomic_umin v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_umin v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_umin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps float @atomic_umin_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.umin.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_smax_1d:
 ; GFX6789: image_atomic_smax v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_smax v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_smax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps float @atomic_smax_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.smax.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_umax_1d:
 ; GFX6789: image_atomic_umax v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_umax v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_umax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps float @atomic_umax_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.umax.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_and_1d:
 ; GFX6789: image_atomic_and v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_and v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_and v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps float @atomic_and_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.and.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_or_1d:
 ; GFX6789: image_atomic_or v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_or v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_or v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps float @atomic_or_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.or.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_xor_1d:
 ; GFX6789: image_atomic_xor v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_xor v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_xor v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps float @atomic_xor_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.xor.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_inc_1d:
 ; GFX6789: image_atomic_inc v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_inc v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_inc v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps float @atomic_inc_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.inc.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_dec_1d:
 ; GFX6789: image_atomic_dec v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_dec v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_dec v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps float @atomic_dec_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.dec.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_cmpswap_1d:
 ; GFX6789: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc{{$}}
+; GFX90A: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc{{$}}
 ; GFX10: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps float @atomic_cmpswap_1d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %swap, i32 %s) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32(i32 %cmp, i32 %swap, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_cmpswap_1d_64:
 ; GFX6789: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc{{$}}
+; GFX90A: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc{{$}}
 ; GFX10: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc ;
 define amdgpu_ps <2 x float> @atomic_cmpswap_1d_64(<8 x i32> inreg %rsrc, i64 %cmp, i64 %swap, i32 %s) {
 main_body:
   %v = call i64 @llvm.amdgcn.image.atomic.cmpswap.1d.i64.i32(i64 %cmp, i64 %swap, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i64 %v to <2 x float>
   ret <2 x float> %out
 }
 
 ; GCN-LABEL: {{^}}atomic_add_2d:
 ; GFX6789: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_add v0, v[{{[02468]}}:{{[13579]}}], s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm glc ;
 define amdgpu_ps float @atomic_add_2d(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.add.2d.i32.i32(i32 %data, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_add_3d:
 ; GFX6789: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_add v0, v[{{[02468]}}:{{[02468]}}], s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm glc ;
 define amdgpu_ps float @atomic_add_3d(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %r) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.add.3d.i32.i32(i32 %data, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_add_cube:
 ; GFX6789: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc da{{$}}
+; GFX90A: image_atomic_add v0, v[{{[02468]}}:{{[02468]}}], s[0:7] dmask:0x1 unorm glc da{{$}}
 ; GFX10: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE unorm glc ;
 define amdgpu_ps float @atomic_add_cube(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %face) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.add.cube.i32.i32(i32 %data, i32 %s, i32 %t, i32 %face, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_add_1darray:
 ; GFX6789: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc da{{$}}
+; GFX90A: image_atomic_add v0, v[{{[02468]}}:{{[13579]}}], s[0:7] dmask:0x1 unorm glc da{{$}}
 ; GFX10: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc ;
 define amdgpu_ps float @atomic_add_1darray(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %slice) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.add.1darray.i32.i32(i32 %data, i32 %s, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_add_2darray:
 ; GFX6789: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc da{{$}}
+; GFX90A: image_atomic_add v0, v[{{[02468]}}:{{[02468]}}], s[0:7] dmask:0x1 unorm glc da{{$}}
 ; GFX10: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc ;
 define amdgpu_ps float @atomic_add_2darray(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %slice) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.add.2darray.i32.i32(i32 %data, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_add_2dmsaa:
 ; GFX6789: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_add v0, v[{{[02468]}}:{{[02468]}}], s[0:7] dmask:0x1 unorm glc{{$}}
 ; GFX10: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm glc ;
 define amdgpu_ps float @atomic_add_2dmsaa(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %fragid) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.add.2dmsaa.i32.i32(i32 %data, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_add_2darraymsaa:
 ; GFX6789: image_atomic_add v0, v[1:4], s[0:7] dmask:0x1 unorm glc da{{$}}
+; GFX90A: image_atomic_add v0, v[{{[02468]}}:{{[13579]}}], s[0:7] dmask:0x1 unorm glc da{{$}}
 ; GFX10: image_atomic_add v0, v[1:4], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc ;
 define amdgpu_ps float @atomic_add_2darraymsaa(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.add.2darraymsaa.i32.i32(i32 %data, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   %out = bitcast i32 %v to float
   ret float %out
 }
 
 ; GCN-LABEL: {{^}}atomic_add_1d_slc:
 ; GFX6789: image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc slc{{$}}
+; GFX90A: image_atomic_add v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc slc{{$}}
 ; GFX10: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc slc ;
 define amdgpu_ps float @atomic_add_1d_slc(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.add.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 2)
   %out = bitcast i32 %v to float
   ret float %out
 }
 

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll

 ; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
 
 ; GCN-LABEL: {{^}}load_1d:
 ; GCN: image_load v[0:3], v0, s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_1d(<8 x i32> inreg %rsrc, i32 %s) {
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }

 ; GCN-LABEL: {{^}}load_2darraymsaa:
 ; GCN: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.2darraymsaa.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
+; GCN-LABEL: {{^}}load_1d_addr_align:
+; GCN: v_mov_b32_e32 [[VADDR:v[0-9]?[02468]]], v1
+; GCN: image_load v[0:3], [[VADDR]], s[0:7] dmask:0xf unorm{{$}}
+define amdgpu_ps <4 x float> @load_1d_addr_align(<8 x i32> inreg %rsrc, <2 x i32> %s) {
+main_body:
+  %s1 = extractelement <2 x i32> %s, i32 1
+  %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s1, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
 ; GCN-LABEL: {{^}}store_1d:
 ; GCN: image_store v[0:3], v4, s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps void @store_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s) {
 main_body:
   call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 

 ; GCN: image_load v[0:1], v0, s[0:7] dmask:0x9 unorm{{$}}
 define amdgpu_ps <2 x float> @load_1d_V2(<8 x i32> inreg %rsrc, i32 %s) {
 main_body:
   %v = call <2 x float> @llvm.amdgcn.image.load.1d.v2f32.i32(i32 9, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret <2 x float> %v
 }
 
 ; GCN-LABEL: {{^}}store_1d_V1:
-; GCN: image_store v0, v1, s[0:7] dmask:0x2 unorm{{$}}
+; GCN: v_mov_b32_e32 [[VADDR:v[0-9]?[02468]]], v1
+; GCN: image_store v0, [[VADDR]], s[0:7] dmask:0x2 unorm{{$}}
 define amdgpu_ps void @store_1d_V1(<8 x i32> inreg %rsrc, float %vdata, i32 %s) {
 main_body:
   call void @llvm.amdgcn.image.store.1d.f32.i32(float %vdata, i32 2, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
 ; GCN-LABEL: {{^}}store_1d_V2:
 ; GCN: image_store v[0:1], v2, s[0:7] dmask:0xc unorm{{$}}

 main_body:
   call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %arg3, i32 15, i32 %arg4, <8 x i32> %arg, i32 0, i32 0)
   %data = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %arg4, <8 x i32> %arg1, i32 0, i32 0)
   call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %data, i32 15, i32 %arg4, <8 x i32> %arg2, i32 0, i32 0)
   ret void
 }
 
 ; GCN-LABEL: image_load_mmo
-; GCN: image_load v1, v[2:3], s[0:7] dmask:0x1 unorm
+; GCN: image_load v1, v[{{[0-9:]+}}], s[0:7] dmask:0x1 unorm
 define amdgpu_ps float @image_load_mmo(<8 x i32> inreg %rsrc, float addrspace(3)* %lds, <2 x i32> %c) #0 {
   store float 0.000000e+00, float addrspace(3)* %lds
   %c0 = extractelement <2 x i32> %c, i32 0
   %c1 = extractelement <2 x i32> %c, i32 1
   %tex = call float @llvm.amdgcn.image.load.2d.f32.i32(i32 1, i32 %c0, i32 %c1, <8 x i32> %rsrc, i32 0, i32 0)
   %tmp2 = getelementptr float, float addrspace(3)* %lds, i32 4
   store float 0.000000e+00, float addrspace(3)* %tmp2
   ret float %tex
 }
 
+; GCN: v_mov_b32_e32 [[VADDR:v[0-9]?[02468]]], v1
+; GCN: image_get_resinfo v[0:3], [[VADDR]], s[0:7] dmask:0xf unorm
+define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, <2 x i32> %s) {
+main_body:
+  %s1 = extractelement <2 x i32> %s, i32 1
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32 15, i32 %s1, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
 declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #1
 declare {float,i32} @llvm.amdgcn.image.load.1d.f32i32.i32(i32, i32, <8 x i32>, i32, i32) #1
 declare {<2 x float>,i32} @llvm.amdgcn.image.load.1d.v2f32i32.i32(i32, i32, <8 x i32>, i32, i32) #1
 declare {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32, i32, <8 x i32>, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1
 declare {<4 x float>,i32} @llvm.amdgcn.image.load.2d.v4f32i32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
 declare {<4 x float>,i32} @llvm.amdgcn.image.load.3d.v4f32i32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1

 declare void @llvm.amdgcn.image.store.2darraymsaa.v4f32.i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
 
 declare float @llvm.amdgcn.image.load.1d.f32.i32(i32, i32, <8 x i32>, i32, i32) #1
 declare float @llvm.amdgcn.image.load.2d.f32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1
 declare <2 x float> @llvm.amdgcn.image.load.1d.v2f32.i32(i32, i32, <8 x i32>, i32, i32) #1
 declare void @llvm.amdgcn.image.store.1d.f32.i32(float, i32, i32, <8 x i32>, i32, i32) #0
 declare void @llvm.amdgcn.image.store.1d.v2f32.i32(<2 x float>, i32, i32, <8 x i32>, i32, i32) #0
 
+declare <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #2
+
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readonly }
 attributes #2 = { nounwind readnone }

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.gfx90a.ll

-; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A %s
+; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A,SDAG %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A,GISEL %s
 
 ; GFX90A-LABEL: {{^}}sample_1d:
 ; GFX90A-NOT: s_wqm_b64
 ; GFX90A:     image_sample v[{{[0-9:]+}}], v{{[0-9]+}}, s[{{[0-9:]+}}], s[{{[0-9:]+}}] dmask:0xf
 define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   ret <4 x float> %v

 ; GFX90A-NOT: s_wqm_b64
 ; GFX90A:     image_sample v[{{[0-9:]+}}], v{{[0-9]+}}, s[{{[0-9:]+}}], s[{{[0-9:]+}}] dmask:0xf unorm
 define amdgpu_ps <4 x float> @sample_1d_unorm(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 1, i32 0, i32 0)
   ret <4 x float> %v
 }
 
+; Address register must be even aligned.
+
+; GFX90A-LABEL: {{^}}sample_1d_addr_align:
+; GFX90A: v_mov_b32_e32 [[VADDR:v[0-9]?[02468]]], v1
+; SDAG:   image_sample v{{[0-9]+}}, [[VADDR]], s[{{[0-9:]+}}], s[{{[0-9:]+}}] dmask:0x1
+; GISEL:  image_sample v[{{[0-9:]+}}], [[VADDR]], s[{{[0-9:]+}}], s[{{[0-9:]+}}] dmask:0xf
+define amdgpu_ps float @sample_1d_addr_align(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, <2 x float> %s) {
+main_body:
+  %s1 = extractelement <2 x float> %s, i32 1
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s1, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v1 = extractelement <4 x float> %v, i32 0
+  ret float %v1
+}
+
 declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32)
 declare {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32)
 declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32)
 declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32)
 declare <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32)
 declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32)

llvm/test/CodeGen/AMDGPU/verify-image-vaddr-align.mir

+# RUN: not --crash llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx90a -run-pass=machineverifier -o /dev/null %s 2>&1 | FileCheck -check-prefix=GFX90A-ERR %s
+
+# GFX90A-ERR: *** Bad machine code: Subtarget requires even aligned vector registers for vaddr operand of image instructions ***
+# GFX90A-ERR: %4:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx90a %0.sub1:vreg_128_align2
+# GFX90A-ERR: *** Bad machine code: Subtarget requires even aligned vector registers for vaddr operand of image instructions ***
+# GFX90A-ERR: $vgpr0 = IMAGE_SAMPLE_V1_V1_gfx90a $vgpr1,
+# GFX90A-ERR: *** Bad machine code: Subtarget requires even aligned vector registers for vaddr operand of image instructions ***
+# GFX90A-ERR: %5:vgpr_32 = IMAGE_LOAD_V1_V1_gfx90a %0.sub1:vreg_128_align2
+# GFX90A-ERR: *** Bad machine code: Subtarget requires even aligned vector registers for vaddr operand of image instructions ***
+# GFX90A-ERR: IMAGE_STORE_V1_V1_gfx90a $vgpr1,
+# GFX90A-ERR: *** Bad machine code: Subtarget requires even aligned vector registers for vaddr operand of image instructions ***
+# GFX90A-ERR: %6:vgpr_32 = IMAGE_ATOMIC_SWAP_V1_V1_gfx90a %6:vgpr_32(tied-def 0), $vgpr1,
+---
+name:            image_sample_odd_vgpr
+body:             |
+  bb.0:
+    %0:vreg_128_align2 = IMPLICIT_DEF
+    %1:areg_128_align2 = IMPLICIT_DEF
+    %2:sgpr_256 = IMPLICIT_DEF
+    %3:sgpr_128 = IMPLICIT_DEF
+
+    %4:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx90a %0.sub1, %2, %3, 1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from custom "ImageResource")
+    $vgpr0 = IMAGE_SAMPLE_V1_V1_gfx90a $vgpr1, %2, %3, 1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from custom "ImageResource")
+    %5:vgpr_32 = IMAGE_LOAD_V1_V1_gfx90a %0.sub1, %2, 8, -1, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from custom "ImageResource")
+    IMAGE_STORE_V1_V1_gfx90a $vgpr1, %5, %2, 2, -1, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32) into custom "ImageResource")
+    %6:vgpr_32 = IMAGE_ATOMIC_SWAP_V1_V1_gfx90a %6:vgpr_32, $vgpr1, %2, 1, -1, 1, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on custom "ImageResource")
+...