[AMDGPU] flat scratch SVS addressing mode for gfx940

[rampitec]

Summary

Both VADDR and SADDR are used in SVS mode.

[foad]

Looks OK.

[rampitec]

Added one more test.

llvm/lib/Target/AMDGPU/AMDGPUGISel.td

 def gi_flat_scratch_offset :
     GIComplexOperandMatcher<s32, "selectScratchOffset">,
     GIComplexPatternEquiv<ScratchOffset>;
 
 def gi_flat_scratch_saddr :
     GIComplexOperandMatcher<s32, "selectScratchSAddr">,
     GIComplexPatternEquiv<ScratchSAddr>;
 
+def gi_flat_scratch_svaddr :
+    GIComplexOperandMatcher<s32, "selectScratchSVAddr">,
+    GIComplexPatternEquiv<ScratchSVAddr>;
+
 def gi_ds_1addr_1offset :
     GIComplexOperandMatcher<s32, "selectDS1Addr1Offset">,
     GIComplexPatternEquiv<DS1Addr1Offset>;
 
 def gi_ds_64bit_4byte_aligned :
     GIComplexOperandMatcher<s64, "selectDS64Bit4ByteAligned">,
     GIComplexPatternEquiv<DS64Bit4ByteAligned>;
 

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h

private:
   bool SelectGlobalOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
                           SDValue &Offset) const;
   bool SelectScratchOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
                            SDValue &Offset) const;
   bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
                          SDValue &VOffset, SDValue &Offset) const;
   bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
                           SDValue &Offset) const;
+  bool SelectScratchSVAddr(SDNode *N, SDValue Addr, SDValue &VAddr,
+                           SDValue &SAddr, SDValue &Offset) const;
 
   bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset,
                         bool &Imm) const;
   SDValue Expand32BitAddress(SDValue Addr) const;
   bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
                   bool &Imm) const;
   bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
   bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_I32, DL, MVT::i32,
                     0);
   }
 
   Offset = CurDAG->getTargetConstant(COffsetVal, DL, MVT::i16);
 
   return true;
 }
 
+bool AMDGPUDAGToDAGISel::SelectScratchSVAddr(SDNode *N, SDValue Addr,
+                                             SDValue &VAddr, SDValue &SAddr,
+                                             SDValue &Offset) const  {
+  int64_t ImmOffset = 0;
+
+  SDValue LHS, RHS;
+  if (isBaseWithConstantOffset64(Addr, LHS, RHS)) {
+    int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue();
+    const SIInstrInfo *TII = Subtarget->getInstrInfo();
+
+    if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, true)) {
+      Addr = LHS;
+      ImmOffset = COffsetVal;
+    } else if (!LHS->isDivergent() && COffsetVal > 0) {
+      SDLoc SL(N);
+      // saddr + large_offset -> saddr + (vaddr = large_offset & ~MaxOffset) +
+      //                         (large_offset & MaxOffset);
+      int64_t SplitImmOffset, RemainderOffset;
+      std::tie(SplitImmOffset, RemainderOffset)
+        = TII->splitFlatOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, true);
+
+      if (isUInt<32>(RemainderOffset)) {
+        SDNode *VMov = CurDAG->getMachineNode(
+          AMDGPU::V_MOV_B32_e32, SL, MVT::i32,
+          CurDAG->getTargetConstant(RemainderOffset, SDLoc(), MVT::i32));
+        VAddr = SDValue(VMov, 0);
+        SAddr = LHS;
+        Offset = CurDAG->getTargetConstant(SplitImmOffset, SDLoc(), MVT::i16);
+        return true;
+      }
+    }
+  }
+
+  if (Addr.getOpcode() != ISD::ADD)
+    return false;
+
+  LHS = Addr.getOperand(0);
+  RHS = Addr.getOperand(1);
+
+  if (!LHS->isDivergent() && RHS->isDivergent()) {
+    SAddr = LHS;
+    VAddr = RHS;
+  } else if (!RHS->isDivergent() && LHS->isDivergent()) {
+    SAddr = RHS;
+    VAddr = LHS;
+  } else {
+    return false;
+  }
+
+  SAddr = SelectSAddrFI(CurDAG, SAddr);
+  Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16);
+  return true;
+}
+
 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
                                           SDValue &Offset, bool &Imm) const {
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
   if (!C) {
     if (ByteOffsetNode.getValueType().isScalarInteger() &&
         ByteOffsetNode.getValueType().getSizeInBits() == 32) {
       Offset = ByteOffsetNode;
       Imm = false;

llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h

private:
   InstructionSelector::ComplexRendererFns
   selectScratchOffset(MachineOperand &Root) const;
 
   InstructionSelector::ComplexRendererFns
   selectGlobalSAddr(MachineOperand &Root) const;
 
   InstructionSelector::ComplexRendererFns
   selectScratchSAddr(MachineOperand &Root) const;
+  InstructionSelector::ComplexRendererFns
+  selectScratchSVAddr(MachineOperand &Root) const;
 
   InstructionSelector::ComplexRendererFns
   selectMUBUFScratchOffen(MachineOperand &Root) const;
   InstructionSelector::ComplexRendererFns
   selectMUBUFScratchOffset(MachineOperand &Root) const;
 
   bool isDSOffsetLegal(Register Base, int64_t Offset) const;
   bool isDSOffset2Legal(Register Base, int64_t Offset0, int64_t Offset1,

llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

AMDGPUInstructionSelector::selectScratchSAddr(MachineOperand &Root) const {
 
   return {{
       [=](MachineInstrBuilder &MIB) { MIB.addReg(SAddr); }, // saddr
       [=](MachineInstrBuilder &MIB) { MIB.addImm(ImmOffset); } // offset
   }};
 }
 
 InstructionSelector::ComplexRendererFns
+AMDGPUInstructionSelector::selectScratchSVAddr(MachineOperand &Root) const {
+  Register Addr = Root.getReg();
+  Register PtrBase;
+  int64_t ConstOffset;
+  int64_t ImmOffset = 0;
+
+  // Match the immediate offset first, which canonically is moved as low as
+  // possible.
+  std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI);
+
+  if (ConstOffset != 0 &&
+      TII.isLegalFLATOffset(ConstOffset, AMDGPUAS::PRIVATE_ADDRESS, true)) {
+    Addr = PtrBase;
+    ImmOffset = ConstOffset;
+  }
+
+  auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI);
+  if (!AddrDef)
+    return None;
+
+  if (AddrDef->MI->getOpcode() != AMDGPU::G_PTR_ADD)
+    return None;
+
+  Register RHS = AddrDef->MI->getOperand(2).getReg();
+  if (RBI.getRegBank(RHS, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
+    return None;
+
+  Register LHS = AddrDef->MI->getOperand(1).getReg();
+  auto LHSDef = getDefSrcRegIgnoringCopies(LHS, *MRI);
+
+  if (LHSDef && LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
+    int FI = LHSDef->MI->getOperand(1).getIndex();
+    return {{
+        [=](MachineInstrBuilder &MIB) { MIB.addReg(RHS); }, // vaddr
+        [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(FI); }, // saddr
+        [=](MachineInstrBuilder &MIB) { MIB.addImm(ImmOffset); } // offset
+    }};
+  }
+
+  if (!isSGPR(LHS))
+    return None;
+
+  return {{
+      [=](MachineInstrBuilder &MIB) { MIB.addReg(RHS); }, // vaddr
+      [=](MachineInstrBuilder &MIB) { MIB.addReg(LHS); }, // saddr
+      [=](MachineInstrBuilder &MIB) { MIB.addImm(ImmOffset); } // offset
+  }};
+}
+
+InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
   MachineInstr *MI = Root.getParent();
   MachineBasicBlock *MBB = MI->getParent();
   MachineFunction *MF = MBB->getParent();
   const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
 
   int64_t Offset = 0;
   if (mi_match(Root.getReg(), *MRI, m_ICst(Offset)) &&

llvm/lib/Target/AMDGPU/FLATInstructions.td

 //===-- FLATInstructions.td - FLAT Instruction Definitions ----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 def FlatOffset : ComplexPattern<iPTR, 2, "SelectFlatOffset", [], [SDNPWantRoot], -10>;
 def GlobalOffset : ComplexPattern<iPTR, 2, "SelectGlobalOffset", [], [SDNPWantRoot], -10>;
 def ScratchOffset : ComplexPattern<iPTR, 2, "SelectScratchOffset", [], [SDNPWantRoot], -10>;
 
 def GlobalSAddr : ComplexPattern<iPTR, 3, "SelectGlobalSAddr", [], [SDNPWantRoot], -10>;
 def ScratchSAddr : ComplexPattern<iPTR, 2, "SelectScratchSAddr", [], [SDNPWantRoot], -10>;
+def ScratchSVAddr : ComplexPattern<iPTR, 3, "SelectScratchSVAddr", [], [SDNPWantRoot], -10>;
 
 //===----------------------------------------------------------------------===//
 // FLAT classes
 //===----------------------------------------------------------------------===//
 
 class FLAT_Pseudo<string opName, dag outs, dag ins,
                   string asmOps, list<dag> pattern=[]> :
   InstSI<outs, ins, "", pattern>,
class FLAT_Pseudo<string opName, dag outs, dag ins,
 
   bits<1> has_data = 1;
   bits<1> has_glc  = 1;
   bits<1> glcValue = 0;
   bits<1> has_dlc  = 1;
   bits<1> dlcValue = 0;
   bits<1> has_sccb  = 1;
   bits<1> sccbValue = 0;
+  bits<1> has_sve  = 0; // Scratch VGPR Enable
+  bits<1> sve = 0;
 
   let SubtargetPredicate = !if(is_flat_global, HasFlatGlobalInsts,
     !if(is_flat_scratch, HasFlatScratchInsts, HasFlatAddressSpace));
 
   // TODO: M0 if it could possibly access LDS (before gfx9? only)?
   let Uses = !if(is_flat_global, [EXEC], [EXEC, FLAT_SCR]);
 
   // Internally, FLAT instruction are executed as both an LDS and a
class FLAT_Real <bits<7> op, FLAT_Pseudo ps> :
   // GFX90A+ only: instruction uses AccVGPR for data
   bits<1> acc = !if(ps.has_vdst, vdst{9}, !if(ps.has_data, vdata{9}, 0));
 
   // We don't use tfe right now, and it was removed in gfx9.
   bits<1> tfe = 0;
 
   // Only valid on GFX9+
   let Inst{12-0} = offset;
-  let Inst{13} = lds;
+  let Inst{13} = !if(ps.has_sve, ps.sve, lds);
   let Inst{15-14} = seg;
 
   let Inst{16}    = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glcValue);
   let Inst{17}    = cpol{CPolBit.SLC};
   let Inst{24-18} = op;
   let Inst{31-26} = 0x37; // Encoding.
   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
   let Inst{47-40} = !if(ps.has_data, vdata{7-0}, ?);

 class FlatScratchInst <string sv_op, string mode> {
   string SVOp = sv_op;
   string Mode = mode;
 }
 
 class FLAT_Scratch_Load_Pseudo <string opName, RegisterClass regClass,
   bit HasTiedOutput = 0,
   bit EnableSaddr = 0,
-  bit EnableVaddr = !not(EnableSaddr)>
+  bit EnableSVE = 0,
+  bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr))>
   : FLAT_Pseudo<
   opName,
   (outs getLdStRegisterOperand<regClass>.ret:$vdst),
   !con(
+    !if(EnableSVE,
+        (ins VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset),
         !if(EnableSaddr,
           (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset),
           !if(EnableVaddr,
             (ins VGPR_32:$vaddr, flat_offset:$offset),
-         (ins flat_offset:$offset))),
+            (ins flat_offset:$offset)))),
      !if(HasTiedOutput, (ins CPol:$cpol, getLdStRegisterOperand<regClass>.ret:$vdst_in),
                         (ins CPol_0:$cpol))),
   " $vdst, "#!if(EnableVaddr, "$vaddr, ", "off, ")#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> {
   let has_data = 0;
   let mayLoad = 1;
   let has_saddr = 1;
   let enabled_saddr = EnableSaddr;
   let has_vaddr = EnableVaddr;
-  let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST"));
+  let has_sve = EnableSVE;
+  let sve = EnableVaddr;
+  let PseudoInstr = opName#!if(EnableSVE, "_SVS", !if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST")));
   let maybeAtomic = 1;
 
   let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", "");
   let DisableEncoding = !if(HasTiedOutput, "$vdst_in", "");
 }
 
 class FLAT_Scratch_Store_Pseudo <string opName, RegisterClass vdataClass, bit EnableSaddr = 0,
-  bit EnableVaddr = !not(EnableSaddr),
+  bit EnableSVE = 0,
+  bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr)),
   RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> : FLAT_Pseudo<
   opName,
   (outs),
+  !if(EnableSVE,
+    (ins vdata_op:$vdata, VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol_0:$cpol),
     !if(EnableSaddr,
       (ins vdata_op:$vdata, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol_0:$cpol),
       !if(EnableVaddr,
         (ins vdata_op:$vdata, VGPR_32:$vaddr, flat_offset:$offset, CPol_0:$cpol),
-      (ins vdata_op:$vdata, flat_offset:$offset, CPol_0:$cpol))),
+        (ins vdata_op:$vdata, flat_offset:$offset, CPol_0:$cpol)))),
   " "#!if(EnableVaddr, "$vaddr", "off")#", $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> {
   let mayLoad  = 0;
   let mayStore = 1;
   let has_vdst = 0;
   let has_saddr = 1;
   let enabled_saddr = EnableSaddr;
   let has_vaddr = EnableVaddr;
-  let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST"));
+  let has_sve = EnableSVE;
+  let sve = EnableVaddr;
+  let PseudoInstr = opName#!if(EnableSVE, "_SVS", !if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST")));
   let maybeAtomic = 1;
 }
 
 multiclass FLAT_Scratch_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedOutput = 0> {
   let is_flat_scratch = 1 in {
     def "" : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput>,
              FlatScratchInst<opName, "SV">;
     def _SADDR : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1>,
                  FlatScratchInst<opName, "SS">;
 
+    let SubtargetPredicate = isGFX940Plus in
+    def _SVS : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1, 1>,
+               FlatScratchInst<opName, "SVS">;
+
     let SubtargetPredicate = HasFlatScratchSTMode in
-    def _ST  : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 0, 0>,
+    def _ST  : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 0, 0, 0>,
                FlatScratchInst<opName, "ST">;
   }
 }
 
 multiclass FLAT_Scratch_Store_Pseudo<string opName, RegisterClass regClass> {
   let is_flat_scratch = 1 in {
     def "" : FLAT_Scratch_Store_Pseudo<opName, regClass>,
              FlatScratchInst<opName, "SV">;
     def _SADDR : FLAT_Scratch_Store_Pseudo<opName, regClass, 1>,
                  FlatScratchInst<opName, "SS">;
 
+    let SubtargetPredicate = isGFX940Plus in
+    def _SVS : FLAT_Scratch_Store_Pseudo<opName, regClass, 1, 1>,
+               FlatScratchInst<opName, "SVS">;
+
     let SubtargetPredicate = HasFlatScratchSTMode in
-    def _ST  : FLAT_Scratch_Store_Pseudo<opName, regClass, 0, 0>,
+    def _ST  : FLAT_Scratch_Store_Pseudo<opName, regClass, 0, 0, 0>,
                FlatScratchInst<opName, "ST">;
   }
 }
 
 class FLAT_AtomicNoRet_Pseudo<string opName, dag outs, dag ins,
                                string asm, list<dag> pattern = []> :
   FLAT_Pseudo<opName, outs, ins, asm, pattern> {
     let mayLoad = 1;

 >;
 
 class ScratchStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node,
                             ValueType vt> : GCNPat <
   (node vt:$data, (ScratchSAddr (i32 SGPR_32:$saddr), i16:$offset)),
   (inst getVregSrcForVT<vt>.ret:$data, $saddr, $offset)
 >;
 
+class ScratchLoadSVaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <
+  (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i16:$offset))),
+  (inst $vaddr, $saddr, $offset, 0)
+>;
+
+class ScratchStoreSVaddrPat <FLAT_Pseudo inst, SDPatternOperator node,
+                             ValueType vt> : GCNPat <
+  (node vt:$data, (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i16:$offset)),
+  (inst getVregSrcForVT<vt>.ret:$data, $vaddr, $saddr, $offset)
+>;
+
+class ScratchLoadSVaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <
+  (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i16:$offset), vt:$in)),
+  (inst $vaddr, $saddr, $offset, 0, $in)
+>;
+
 let OtherPredicates = [HasFlatAddressSpace] in {
 
 def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i32>;
 def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i32>;
 def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>;
 def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i16>;
 def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>;
 def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>;

 multiclass ScratchFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {
   def : ScratchLoadSignedPat <inst, node, vt> {
     let AddedComplexity = 25;
   }
 
   def : ScratchLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {
     let AddedComplexity = 26;
   }
+
+  def : ScratchLoadSVaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> {
+    let SubtargetPredicate = isGFX940Plus;
+    let AddedComplexity = 27;
+  }
 }
 
 multiclass ScratchFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node,
                                ValueType vt> {
   def : ScratchStoreSignedPat <inst, node, vt> {
     let AddedComplexity = 25;
   }
 
   def : ScratchStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {
     let AddedComplexity = 26;
   }
+
+  def : ScratchStoreSVaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> {
+    let SubtargetPredicate = isGFX940Plus;
+    let AddedComplexity = 27;
+  }
 }
 
 multiclass ScratchFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {
   def : ScratchLoadSignedPat_D16 <inst, node, vt> {
     let AddedComplexity = 25;
   }
 
   def : ScratchLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {
     let AddedComplexity = 26;
   }
+
+  def : ScratchLoadSVaddrPat_D16 <!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> {
+    let SubtargetPredicate = isGFX940Plus;
+    let AddedComplexity = 27;
+  }
 }
 
 let OtherPredicates = [HasFlatGlobalInsts] in {
 
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i32>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i32>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i32>;
 defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i16>;
multiclass FLAT_Real_AllAddr_vi<bits<7> op,
   def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR"), has_sccb>;
 }
 
 class FLAT_Real_gfx940 <bits<7> op, FLAT_Pseudo ps> :
   FLAT_Real <op, ps>,
   SIMCInstr <ps.PseudoInstr, SIEncodingFamily.GFX940> {
   let AssemblerPredicate = isGFX940Plus;
   let DecoderNamespace = "GFX9";
+  let Inst{13} = ps.sve;
   let Inst{25} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccbValue);
 }
 
+multiclass FLAT_Real_AllAddr_SVE_vi<bits<7> op> {
+  def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME)> {
+    let AssemblerPredicate = isGFX8GFX9NotGFX940;
+    let OtherPredicates = [isGFX8GFX9NotGFX940];
+  }
+  def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")> {
+    let DecoderNamespace = "GFX9";
+  }
+  let AssemblerPredicate = isGFX940Plus, SubtargetPredicate = isGFX940Plus in {
+    def _VE_gfx940  : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME)>;
+    def _SVS_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SVS")>;
+    def _ST_gfx940  : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_ST")>;
+  }
+}
+
 def FLAT_LOAD_UBYTE_vi         : FLAT_Real_vi <0x10, FLAT_LOAD_UBYTE>;
 def FLAT_LOAD_SBYTE_vi         : FLAT_Real_vi <0x11, FLAT_LOAD_SBYTE>;
 def FLAT_LOAD_USHORT_vi        : FLAT_Real_vi <0x12, FLAT_LOAD_USHORT>;
 def FLAT_LOAD_SSHORT_vi        : FLAT_Real_vi <0x13, FLAT_LOAD_SSHORT>;
 def FLAT_LOAD_DWORD_vi         : FLAT_Real_vi <0x14, FLAT_LOAD_DWORD>;
 def FLAT_LOAD_DWORDX2_vi       : FLAT_Real_vi <0x15, FLAT_LOAD_DWORDX2>;
 def FLAT_LOAD_DWORDX4_vi       : FLAT_Real_vi <0x17, FLAT_LOAD_DWORDX4>;
 def FLAT_LOAD_DWORDX3_vi       : FLAT_Real_vi <0x16, FLAT_LOAD_DWORDX3>;

 defm GLOBAL_ATOMIC_SMAX_X2    : FLAT_Global_Real_Atomics_vi <0x66>;
 defm GLOBAL_ATOMIC_UMAX_X2    : FLAT_Global_Real_Atomics_vi <0x67>;
 defm GLOBAL_ATOMIC_AND_X2     : FLAT_Global_Real_Atomics_vi <0x68>;
 defm GLOBAL_ATOMIC_OR_X2      : FLAT_Global_Real_Atomics_vi <0x69>;
 defm GLOBAL_ATOMIC_XOR_X2     : FLAT_Global_Real_Atomics_vi <0x6a>;
 defm GLOBAL_ATOMIC_INC_X2     : FLAT_Global_Real_Atomics_vi <0x6b>;
 defm GLOBAL_ATOMIC_DEC_X2     : FLAT_Global_Real_Atomics_vi <0x6c>;
 
-defm SCRATCH_LOAD_UBYTE         : FLAT_Real_AllAddr_vi <0x10>;
-defm SCRATCH_LOAD_SBYTE         : FLAT_Real_AllAddr_vi <0x11>;
-defm SCRATCH_LOAD_USHORT        : FLAT_Real_AllAddr_vi <0x12>;
-defm SCRATCH_LOAD_SSHORT        : FLAT_Real_AllAddr_vi <0x13>;
-defm SCRATCH_LOAD_DWORD         : FLAT_Real_AllAddr_vi <0x14>;
-defm SCRATCH_LOAD_DWORDX2       : FLAT_Real_AllAddr_vi <0x15>;
-defm SCRATCH_LOAD_DWORDX3       : FLAT_Real_AllAddr_vi <0x16>;
-defm SCRATCH_LOAD_DWORDX4       : FLAT_Real_AllAddr_vi <0x17>;
-defm SCRATCH_STORE_BYTE         : FLAT_Real_AllAddr_vi <0x18>;
-defm SCRATCH_STORE_BYTE_D16_HI  : FLAT_Real_AllAddr_vi <0x19>;
-defm SCRATCH_LOAD_UBYTE_D16     : FLAT_Real_AllAddr_vi <0x20>;
-defm SCRATCH_LOAD_UBYTE_D16_HI  : FLAT_Real_AllAddr_vi <0x21>;
-defm SCRATCH_LOAD_SBYTE_D16     : FLAT_Real_AllAddr_vi <0x22>;
-defm SCRATCH_LOAD_SBYTE_D16_HI  : FLAT_Real_AllAddr_vi <0x23>;
-defm SCRATCH_LOAD_SHORT_D16     : FLAT_Real_AllAddr_vi <0x24>;
-defm SCRATCH_LOAD_SHORT_D16_HI  : FLAT_Real_AllAddr_vi <0x25>;
-defm SCRATCH_STORE_SHORT        : FLAT_Real_AllAddr_vi <0x1a>;
-defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>;
-defm SCRATCH_STORE_DWORD        : FLAT_Real_AllAddr_vi <0x1c>;
-defm SCRATCH_STORE_DWORDX2      : FLAT_Real_AllAddr_vi <0x1d>;
-defm SCRATCH_STORE_DWORDX3      : FLAT_Real_AllAddr_vi <0x1e>;
-defm SCRATCH_STORE_DWORDX4      : FLAT_Real_AllAddr_vi <0x1f>;
+defm SCRATCH_LOAD_UBYTE         : FLAT_Real_AllAddr_SVE_vi <0x10>;
+defm SCRATCH_LOAD_SBYTE         : FLAT_Real_AllAddr_SVE_vi <0x11>;
+defm SCRATCH_LOAD_USHORT        : FLAT_Real_AllAddr_SVE_vi <0x12>;
+defm SCRATCH_LOAD_SSHORT        : FLAT_Real_AllAddr_SVE_vi <0x13>;
+defm SCRATCH_LOAD_DWORD         : FLAT_Real_AllAddr_SVE_vi <0x14>;
+defm SCRATCH_LOAD_DWORDX2       : FLAT_Real_AllAddr_SVE_vi <0x15>;
+defm SCRATCH_LOAD_DWORDX3       : FLAT_Real_AllAddr_SVE_vi <0x16>;
+defm SCRATCH_LOAD_DWORDX4       : FLAT_Real_AllAddr_SVE_vi <0x17>;
+defm SCRATCH_STORE_BYTE         : FLAT_Real_AllAddr_SVE_vi <0x18>;
+defm SCRATCH_STORE_BYTE_D16_HI  : FLAT_Real_AllAddr_SVE_vi <0x19>;
+defm SCRATCH_LOAD_UBYTE_D16     : FLAT_Real_AllAddr_SVE_vi <0x20>;
+defm SCRATCH_LOAD_UBYTE_D16_HI  : FLAT_Real_AllAddr_SVE_vi <0x21>;
+defm SCRATCH_LOAD_SBYTE_D16     : FLAT_Real_AllAddr_SVE_vi <0x22>;
+defm SCRATCH_LOAD_SBYTE_D16_HI  : FLAT_Real_AllAddr_SVE_vi <0x23>;
+defm SCRATCH_LOAD_SHORT_D16     : FLAT_Real_AllAddr_SVE_vi <0x24>;
+defm SCRATCH_LOAD_SHORT_D16_HI  : FLAT_Real_AllAddr_SVE_vi <0x25>;
+defm SCRATCH_STORE_SHORT        : FLAT_Real_AllAddr_SVE_vi <0x1a>;
+defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x1b>;
+defm SCRATCH_STORE_DWORD        : FLAT_Real_AllAddr_SVE_vi <0x1c>;
+defm SCRATCH_STORE_DWORDX2      : FLAT_Real_AllAddr_SVE_vi <0x1d>;
+defm SCRATCH_STORE_DWORDX3      : FLAT_Real_AllAddr_SVE_vi <0x1e>;
+defm SCRATCH_STORE_DWORDX4      : FLAT_Real_AllAddr_SVE_vi <0x1f>;
 
 let SubtargetPredicate = isGFX8GFX9NotGFX940 in {
   // These instructions are encoded differently on gfx90* and gfx940.
   defm GLOBAL_ATOMIC_ADD_F32    : FLAT_Global_Real_Atomics_vi <0x04d, 0>;
   defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Real_Atomics_vi <0x04e, 0>;
 }
 
 let SubtargetPredicate = isGFX90AOnly in {

llvm/lib/Target/AMDGPU/SIFoldOperands.cpp

if (TII->isMUBUF(*UseMI)) {
         return;
     }
 
     // A frame index will resolve to a positive constant, so it should always be
     // safe to fold the addressing mode, even pre-GFX9.
     UseMI->getOperand(UseOpIdx).ChangeToFrameIndex(OpToFold.getIndex());
 
     if (TII->isFLATScratch(*UseMI) &&
         AMDGPU::getNamedOperandIdx(UseMI->getOpcode(),
-                                   AMDGPU::OpName::vaddr) != -1) {
+                                   AMDGPU::OpName::vaddr) != -1 &&
+        AMDGPU::getNamedOperandIdx(UseMI->getOpcode(),
+                                   AMDGPU::OpName::saddr) == -1) {
       unsigned NewOpc = AMDGPU::getFlatScratchInstSSfromSV(UseMI->getOpcode());
       UseMI->setDesc(TII->get(NewOpc));
     }
 
     return;
   }
 
   bool FoldingImmLike =

llvm/lib/Target/AMDGPU/SIInstrInfo.h

namespace AMDGPU {
   LLVM_READONLY
   int getVCMPXNoSDstOp(uint16_t Opcode);
 
   /// \returns ST form with only immediate offset of a FLAT Scratch instruction
   /// given an \p Opcode of an SS (SADDR) form.
   LLVM_READONLY
   int getFlatScratchInstSTfromSS(uint16_t Opcode);
 
+  /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
+  /// of an SVS (SADDR + VADDR) form.
+  LLVM_READONLY
+  int getFlatScratchInstSVfromSVS(uint16_t Opcode);
+
   /// \returns SS (SADDR) form of a FLAT Scratch instruction given an \p Opcode
   /// of an SV (VADDR) form.
   LLVM_READONLY
   int getFlatScratchInstSSfromSV(uint16_t Opcode);
 
   /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
   /// of an SS (SADDR) form.
   LLVM_READONLY

llvm/lib/Target/AMDGPU/SIInstrInfo.td

 def getFlatScratchInstSSfromSV : InstrMapping {
   let FilterClass = "FlatScratchInst";
   let RowFields = ["SVOp"];
   let ColFields = ["Mode"];
   let KeyCol = ["SV"];
   let ValueCols = [["SS"]];
 }
 
+def getFlatScratchInstSVfromSVS : InstrMapping {
+  let FilterClass = "FlatScratchInst";
+  let RowFields = ["SVOp"];
+  let ColFields = ["Mode"];
+  let KeyCol = ["SVS"];
+  let ValueCols = [["SV"]];
+}
+
 def getFlatScratchInstSVfromSS : InstrMapping {
   let FilterClass = "FlatScratchInst";
   let RowFields = ["SVOp"];
   let ColFields = ["Mode"];
   let KeyCol = ["SS"];
   let ValueCols = [["SV"]];
 }
 

llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp

default: {
           if (TII->isLegalFLATOffset(NewOffset, AMDGPUAS::PRIVATE_ADDRESS,
                                      SIInstrFlags::FlatScratch)) {
             OffsetOp->setImm(NewOffset);
             if (FrameReg)
               return;
             Offset = 0;
           }
 
-          assert(!TII->getNamedOperand(*MI, AMDGPU::OpName::vaddr) &&
-                 "Unexpected vaddr for flat scratch with a FI operand");
-
+          if (!Offset) {
+            unsigned Opc = MI->getOpcode();
+            int NewOpc = -1;
+            if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr) != -1) {
+              NewOpc = AMDGPU::getFlatScratchInstSVfromSVS(Opc);
+            } else if (ST.hasFlatScratchSTMode()) {
               // On GFX10 we have ST mode to use no registers for an address.
               // Otherwise we need to materialize 0 into an SGPR.
-          if (!Offset && ST.hasFlatScratchSTMode()) {
-            unsigned Opc = MI->getOpcode();
-            unsigned NewOpc = AMDGPU::getFlatScratchInstSTfromSS(Opc);
+              NewOpc = AMDGPU::getFlatScratchInstSTfromSS(Opc);
+            }
+
+            if (NewOpc != -1) {
               MI->RemoveOperand(
                   AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr));
               MI->setDesc(TII->get(NewOpc));
               return;
             }
           }
+        }
 
         if (!FrameReg) {
           FIOp.ChangeToImmediate(Offset);
           if (TII->isImmOperandLegal(*MI, FIOperandNum, FIOp))
             return;
         }
 
         // We need to use register here. Check if we can use an SGPR or need

llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch-init.ll

 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,RW-FLAT %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+architected-flat-scratch < %s | FileCheck -check-prefixes=GCN,RO-FLAT %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 < %s | FileCheck -check-prefixes=GCN,RO-FLAT %s
 
 ; Make sure flat_scratch_init is set
 
 ; GCN-LABEL: {{^}}stack_object_addrspacecast_in_kernel_no_calls:
 ; RW-FLAT:     s_add_u32 flat_scratch_lo, s4, s7
 ; RW-FLAT:     s_addc_u32 flat_scratch_hi, s5, 0
 ; RO-FLAT-NOT: flat_scratch
 ; GCN:         flat_store_dword

llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -global-isel -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx1030 -global-isel -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -march=amdgcn -mcpu=gfx940 -global-isel -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=GFX940 %s
 
 define amdgpu_kernel void @store_load_sindex_kernel(i32 %idx) {
 ; GFX9-LABEL: store_load_sindex_kernel:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x24
 ; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
 ; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 15

 ; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
 ; GFX10-NEXT:    s_add_i32 s0, s0, 4
 ; GFX10-NEXT:    s_add_i32 s1, s1, 4
 ; GFX10-NEXT:    scratch_store_dword off, v0, s0
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, off, s1 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_endpgm
+;
+; GFX940-LABEL: store_load_sindex_kernel:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v0, 15
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
+; GFX940-NEXT:    s_and_b32 s0, s0, 15
+; GFX940-NEXT:    v_mov_b32_e32 v1, s1
+; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
+; GFX940-NEXT:    scratch_store_dword v1, v0, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, s0
+; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
 bb:
   %i = alloca [32 x float], align 4, addrspace(5)
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
   %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
   store volatile i32 15, i32 addrspace(5)* %i8, align 4
   %i9 = and i32 %idx, 15
   %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9

 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
 ; GFX10-NEXT:    v_add_nc_u32_e32 v0, v2, v0
 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, v2, v1
 ; GFX10-NEXT:    scratch_store_dword v0, v3, off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, v1, off offset:124 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_endpgm
+;
+; GFX940-LABEL: store_load_vindex_kernel:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 15
+; GFX940-NEXT:    v_sub_u32_e32 v0, 0, v0
+; GFX940-NEXT:    scratch_store_dword v1, v2, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:128 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
 bb:
   %i = alloca [32 x float], align 4, addrspace(5)
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
   %i3 = zext i32 %i2 to i64
   %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i2
   %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
   store volatile i32 15, i32 addrspace(5)* %i8, align 4

 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
 ; GFX10-NEXT:    v_add_nc_u32_e32 v0, v2, v0
 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, v2, v1
 ; GFX10-NEXT:    scratch_store_dword v0, v3, off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: store_load_vindex_foo:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 15
+; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX940-NEXT:    scratch_store_dword v1, v2, s32 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    scratch_load_dword v0, v0, s32 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %i = alloca [32 x float], align 4, addrspace(5)
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
   %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
   store volatile i32 15, i32 addrspace(5)* %i8, align 4
   %i9 = and i32 %idx, 15
   %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9

 ; GFX10-LABEL: private_ptr_foo:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0x41200000
 ; GFX10-NEXT:    scratch_store_dword v0, v1, off offset:4
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: private_ptr_foo:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0x41200000
+; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:4
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr inbounds float, float addrspace(5)* %arg, i32 1
   store float 1.000000e+01, float addrspace(5)* %gep, align 4
   ret void
 }
 
 define amdgpu_kernel void @store_load_sindex_small_offset_kernel(i32 %idx) {
 ; GFX9-LABEL: store_load_sindex_small_offset_kernel:
 ; GFX9:       ; %bb.0: ; %bb

 ; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
 ; GFX10-NEXT:    s_addk_i32 s0, 0x104
 ; GFX10-NEXT:    s_addk_i32 s1, 0x104
 ; GFX10-NEXT:    scratch_store_dword off, v0, s0
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, off, s1 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_endpgm
+;
+; GFX940-LABEL: store_load_sindex_small_offset_kernel:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 15
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
+; GFX940-NEXT:    s_and_b32 s0, s0, 15
+; GFX940-NEXT:    v_mov_b32_e32 v1, s1
+; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
+; GFX940-NEXT:    scratch_store_dword v1, v0, off offset:260 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, s0
+; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:260 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
 bb:
   %padding = alloca [64 x i32], align 4, addrspace(5)
   %i = alloca [32 x float], align 4, addrspace(5)
   %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
   %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
   %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*

 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, v2, v1
 ; GFX10-NEXT:    scratch_load_dword v2, off, off offset:4 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    scratch_store_dword v0, v3, off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, v1, off offset:124 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_endpgm
+;
+; GFX940-LABEL: store_load_vindex_small_offset_kernel:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    scratch_load_dword v1, off, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 15
+; GFX940-NEXT:    v_sub_u32_e32 v0, 0, v0
+; GFX940-NEXT:    scratch_store_dword v1, v2, off offset:260 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:384 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
 bb:
   %padding = alloca [64 x i32], align 4, addrspace(5)
   %i = alloca [32 x float], align 4, addrspace(5)
   %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
   %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
   %i3 = zext i32 %i2 to i64

 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, v2, v1
 ; GFX10-NEXT:    scratch_load_dword v2, off, s32 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    scratch_store_dword v0, v3, off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: store_load_vindex_small_offset_foo:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    scratch_load_dword v1, off, s32 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 15
+; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX940-NEXT:    scratch_store_dword v1, v2, s32 offset:256 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    scratch_load_dword v0, v0, s32 offset:256 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %padding = alloca [64 x i32], align 4, addrspace(5)
   %i = alloca [32 x float], align 4, addrspace(5)
   %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
   %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
   %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*

 ; GFX10-NEXT:    s_lshl_b32 s1, s1, 2
 ; GFX10-NEXT:    s_addk_i32 s0, 0x4004
 ; GFX10-NEXT:    s_addk_i32 s1, 0x4004
 ; GFX10-NEXT:    scratch_store_dword off, v0, s0
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, off, s1 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_endpgm
+;
+; GFX940-LABEL: store_load_sindex_large_offset_kernel:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 15
+; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4004
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
+; GFX940-NEXT:    s_and_b32 s0, s0, 15
+; GFX940-NEXT:    v_mov_b32_e32 v1, s1
+; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
+; GFX940-NEXT:    scratch_store_dword v1, v0, vcc_hi sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, s0
+; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4004
+; GFX940-NEXT:    scratch_load_dword v0, v0, vcc_hi sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
 bb:
   %padding = alloca [4096 x i32], align 4, addrspace(5)
   %i = alloca [32 x float], align 4, addrspace(5)
   %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
   %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
   %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*

 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, v2, v1
 ; GFX10-NEXT:    scratch_load_dword v2, off, off offset:4 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    scratch_store_dword v0, v3, off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, v1, off offset:124 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_endpgm
+;
+; GFX940-LABEL: store_load_vindex_large_offset_kernel:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    scratch_load_dword v1, off, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 15
+; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4004
+; GFX940-NEXT:    v_sub_u32_e32 v0, 0, v0
+; GFX940-NEXT:    scratch_store_dword v1, v2, vcc_hi sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4004
+; GFX940-NEXT:    scratch_load_dword v0, v0, vcc_hi offset:124 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
 bb:
   %padding = alloca [4096 x i32], align 4, addrspace(5)
   %i = alloca [32 x float], align 4, addrspace(5)
   %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
   %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
   %i3 = zext i32 %i2 to i64

 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, v2, v1
 ; GFX10-NEXT:    scratch_load_dword v2, off, s32 offset:4 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    scratch_store_dword v0, v3, off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: store_load_vindex_large_offset_foo:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    scratch_load_dword v1, off, s32 offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 15
+; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
+; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX940-NEXT:    scratch_store_dword v1, v2, vcc_hi sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
+; GFX940-NEXT:    scratch_load_dword v0, v0, vcc_hi sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %padding = alloca [4096 x i32], align 4, addrspace(5)
   %i = alloca [32 x float], align 4, addrspace(5)
   %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
   %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
   %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*

 ; GFX10-NEXT:    s_add_i32 s0, s0, 4
 ; GFX10-NEXT:    scratch_store_dword off, v0, off offset:4
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_store_dword off, v1, s0
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, off, s0 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_endpgm
+;
+; GFX940-LABEL: store_load_large_imm_offset_kernel:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    v_mov_b32_e32 v0, 13
+; GFX940-NEXT:    scratch_store_dword off, v0, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0x3e80
+; GFX940-NEXT:    v_mov_b32_e32 v1, 15
+; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
 bb:
   %i = alloca [4096 x i32], align 4, addrspace(5)
   %i1 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 undef
   store volatile i32 13, i32 addrspace(5)* %i1, align 4
   %i7 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
   store volatile i32 15, i32 addrspace(5)* %i7, align 4
   %i10 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
   %i12 = load volatile i32, i32 addrspace(5)* %i10, align 4

 ; GFX10-NEXT:    s_add_i32 s0, s0, vcc_lo
 ; GFX10-NEXT:    scratch_store_dword off, v0, s32 offset:4
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_store_dword off, v1, s0
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, off, s0 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: store_load_large_imm_offset_foo:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 13
+; GFX940-NEXT:    scratch_store_dword off, v0, s32 offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0x3e80
+; GFX940-NEXT:    v_mov_b32_e32 v1, 15
+; GFX940-NEXT:    scratch_store_dword v0, v1, s32 offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dword v0, v0, s32 offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %i = alloca [4096 x i32], align 4, addrspace(5)
   %i1 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 undef
   store volatile i32 13, i32 addrspace(5)* %i1, align 4
   %i7 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
   store volatile i32 15, i32 addrspace(5)* %i7, align 4
   %i10 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
   %i12 = load volatile i32, i32 addrspace(5)* %i10, align 4

 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
 ; GFX10-NEXT:    v_add_nc_u32_e32 v0, 4, v0
 ; GFX10-NEXT:    scratch_store_dword v0, v1, off offset:1024
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_endpgm
+;
+; GFX940-LABEL: store_load_vidx_sidx_offset:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v1, 15
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
+; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:1028 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:1028 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
 bb:
   %alloca = alloca [32 x i32], align 4, addrspace(5)
   %vidx = tail call i32 @llvm.amdgcn.workitem.id.x()
   %add1 = add nsw i32 %sidx, %vidx
   %add2 = add nsw i32 %add1, 256
   %gep = getelementptr inbounds [32 x i32], [32 x i32] addrspace(5)* %alloca, i32 0, i32 %add2
   store volatile i32 15, i32 addrspace(5)* %gep, align 4
   %load = load volatile i32, i32 addrspace(5)* %gep, align 4

 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: store_load_i64_aligned:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 15
+; GFX940-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 bb:
   store volatile i64 15, i64 addrspace(5)* %arg, align 8
   %load = load volatile i64, i64 addrspace(5)* %arg, align 8
   ret void
 }
 
 define void @store_load_i64_unaligned(i64 addrspace(5)* nocapture %arg) {
 ; GFX9-LABEL: store_load_i64_unaligned:

 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: store_load_i64_unaligned:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 15
+; GFX940-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 bb:
   store volatile i64 15, i64 addrspace(5)* %arg, align 1
   %load = load volatile i64, i64 addrspace(5)* %arg, align 1
   ret void
 }
 
 define void @store_load_v3i32_unaligned(<3 x i32> addrspace(5)* nocapture %arg) {
 ; GFX9-LABEL: store_load_v3i32_unaligned:

 ; GFX10-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX10-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: store_load_v3i32_unaligned:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    s_mov_b32 s2, 3
+; GFX940-NEXT:    s_mov_b32 s1, 2
+; GFX940-NEXT:    s_mov_b32 s0, 1
+; GFX940-NEXT:    v_mov_b32_e32 v4, s2
+; GFX940-NEXT:    v_mov_b32_e32 v3, s1
+; GFX940-NEXT:    v_mov_b32_e32 v2, s0
+; GFX940-NEXT:    scratch_store_dwordx3 v0, v[2:4], off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dwordx3 v[0:2], v0, off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 bb:
   store volatile <3 x i32> <i32 1, i32 2, i32 3>, <3 x i32> addrspace(5)* %arg, align 1
   %load = load volatile <3 x i32>, <3 x i32> addrspace(5)* %arg, align 1
   ret void
 }
 
 define void @store_load_v4i32_unaligned(<4 x i32> addrspace(5)* nocapture %arg) {
 ; GFX9-LABEL: store_load_v4i32_unaligned:

 ; GFX10-NEXT:    v_mov_b32_e32 v3, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX10-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: store_load_v4i32_unaligned:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    s_mov_b32 s3, 4
+; GFX940-NEXT:    s_mov_b32 s2, 3
+; GFX940-NEXT:    s_mov_b32 s1, 2
+; GFX940-NEXT:    s_mov_b32 s0, 1
+; GFX940-NEXT:    v_mov_b64_e32 v[4:5], s[2:3]
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[0:1]
+; GFX940-NEXT:    scratch_store_dwordx4 v0, v[2:5], off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dwordx4 v[0:3], v0, off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 bb:
   store volatile <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> addrspace(5)* %arg, align 1
   %load = load volatile <4 x i32>, <4 x i32> addrspace(5)* %arg, align 1
   ret void
 }
 
 declare i32 @llvm.amdgcn.workitem.id.x()

llvm/test/CodeGen/AMDGPU/flat-scratch.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefix=GFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx1030 -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefix=GFX10 %s
 ; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefix=GFX9-PAL %s
+; RUN: llc -march=amdgcn -mcpu=gfx940 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940 %s
 ; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1010 -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-PAL,GFX1010-PAL %s
 ; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1030 -mattr=-promote-alloca -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-PAL,GFX1030-PAL %s
 
 define amdgpu_kernel void @zero_init_kernel() {
 ; GFX9-LABEL: zero_init_kernel:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s0, s3
 ; GFX9-NEXT:    s_mov_b32 s0, 0

 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: zero_init_kernel:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_mov_b32 s0, 0
+; GFX940-NEXT:    s_mov_b32 s1, s0
+; GFX940-NEXT:    s_mov_b32 s2, s0
+; GFX940-NEXT:    s_mov_b32 s3, s0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:64
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:48
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:32
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:16
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX1010-PAL-LABEL: zero_init_kernel:
 ; GFX1010-PAL:       ; %bb.0:
 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1

 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:48
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:32
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GFX940-LABEL: zero_init_foo:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    s_mov_b32 s0, 0
+; GFX940-NEXT:    s_mov_b32 s1, s0
+; GFX940-NEXT:    s_mov_b32 s2, s0
+; GFX940-NEXT:    s_mov_b32 s3, s0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:48
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:32
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX10-PAL-LABEL: zero_init_foo:
 ; GFX10-PAL:       ; %bb.0:
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    s_mov_b32 s0, 0
 ; GFX10-PAL-NEXT:    s_mov_b32 s1, s0
 ; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX10-PAL-NEXT:    s_mov_b32 s3, s0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:48
 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:32
 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: zero_init_foo:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, 0
+; GCN-NEXT:    s_mov_b32 s1, s0
+; GCN-NEXT:    s_mov_b32 s2, s0
+; GCN-NEXT:    s_mov_b32 s3, s0
+; GCN-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GCN-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:48
+; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:32
+; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
+; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca [32 x i16], align 2, addrspace(5)
   %cast = bitcast [32 x i16] addrspace(5)* %alloca to i8 addrspace(5)*
   call void @llvm.memset.p5i8.i64(i8 addrspace(5)* align 2 dereferenceable(64) %cast, i8 0, i64 64, i1 false)
   ret void
 }
 
 define amdgpu_kernel void @store_load_sindex_kernel(i32 %idx) {
 ; GFX9-LABEL: store_load_sindex_kernel:

 ; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_add_i32 s0, s0, 4
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: store_load_sindex_kernel:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v0, 15
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
+; GFX940-NEXT:    s_and_b32 s0, s0, 15
+; GFX940-NEXT:    s_add_i32 s1, s1, 4
+; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
+; GFX940-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_add_i32 s0, s0, 4
+; GFX940-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX10-PAL-LABEL: store_load_sindex_kernel:
 ; GFX10-PAL:       ; %bb.0: ; %bb
 ; GFX10-PAL-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-PAL-NEXT:    s_mov_b32 s4, s0
 ; GFX10-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
 ; GFX10-PAL-NEXT:    s_add_u32 s4, s4, s3
 ; GFX10-PAL-NEXT:    s_addc_u32 s5, s5, 0
 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
 ; GFX10-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 15
 ; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_and_b32 s1, s0, 15
 ; GFX10-PAL-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX10-PAL-NEXT:    s_lshl_b32 s1, s1, 2
 ; GFX10-PAL-NEXT:    s_add_i32 s0, s0, 4
 ; GFX10-PAL-NEXT:    s_add_i32 s1, s1, 4
 ; GFX10-PAL-NEXT:    scratch_store_dword off, v0, s0
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    s_endpgm
+; GCN-LABEL: store_load_sindex_kernel:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v0, 15
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s1, s0, 2
+; GCN-NEXT:    s_and_b32 s0, s0, 15
+; GCN-NEXT:    s_lshl_b32 s0, s0, 2
+; GCN-NEXT:    s_add_u32 s1, 4, s1
+; GCN-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_add_u32 s0, 4, s0
+; GCN-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_endpgm
 bb:
   %i = alloca [32 x float], align 4, addrspace(5)
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
   %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
   store volatile i32 15, i32 addrspace(5)* %i8, align 4
   %i9 = and i32 %idx, 15
   %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9

 ; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_add_i32 s0, s0, 4
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: store_load_sindex_foo:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
+; GFX940-NEXT:    s_and_b32 s0, s0, 15
+; GFX940-NEXT:    s_add_i32 s1, s1, 4
+; GFX940-NEXT:    v_mov_b32_e32 v0, 15
+; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
+; GFX940-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_add_i32 s0, s0, 4
+; GFX940-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX10-PAL-LABEL: store_load_sindex_foo:
 ; GFX10-PAL:       ; %bb.0: ; %bb
 ; GFX10-PAL-NEXT:    s_getpc_b64 s[2:3]
 ; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX10-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX10-PAL-NEXT:    s_add_u32 s2, s2, s1
 ; GFX10-PAL-NEXT:    s_addc_u32 s3, s3, 0
 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 15
 ; GFX10-PAL-NEXT:    s_and_b32 s1, s0, 15
 ; GFX10-PAL-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX10-PAL-NEXT:    s_lshl_b32 s1, s1, 2
 ; GFX10-PAL-NEXT:    s_add_i32 s0, s0, 4
 ; GFX10-PAL-NEXT:    s_add_i32 s1, s1, 4
 ; GFX10-PAL-NEXT:    scratch_store_dword off, v0, s0
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_load_dword v0, off, s1 glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    s_endpgm
+; GCN-LABEL: store_load_sindex_foo:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_lshl_b32 s1, s0, 2
+; GCN-NEXT:    s_and_b32 s0, s0, 15
+; GCN-NEXT:    s_lshl_b32 s0, s0, 2
+; GCN-NEXT:    s_add_u32 s1, 4, s1
+; GCN-NEXT:    v_mov_b32_e32 v0, 15
+; GCN-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_add_u32 s0, 4, s0
+; GCN-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_endpgm
 bb:
   %i = alloca [32 x float], align 4, addrspace(5)
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
   %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
   store volatile i32 15, i32 addrspace(5)* %i8, align 4
   %i9 = and i32 %idx, 15
   %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9

 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
 ; GFX9-PAL-NEXT:    scratch_store_dword v2, v3, off
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    v_sub_u32_e32 v0, v1, v0
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: store_load_vindex_kernel:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v1, 15
+; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_sub_u32_e32 v0, 4, v0
+; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:124 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX10-PAL-LABEL: store_load_vindex_kernel:
 ; GFX10-PAL:       ; %bb.0: ; %bb
 ; GFX10-PAL-NEXT:    s_getpc_b64 s[2:3]
 ; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX10-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX10-PAL-NEXT:    s_add_u32 s2, s2, s1
 ; GFX10-PAL-NEXT:    s_addc_u32 s3, s3, 0
 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 4
 ; GFX10-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, 15
 ; GFX10-PAL-NEXT:    v_add_nc_u32_e32 v2, v1, v0
 ; GFX10-PAL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
 ; GFX10-PAL-NEXT:    scratch_store_dword v2, v3, off
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    s_endpgm
+; GCN-LABEL: store_load_vindex_kernel:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-NEXT:    v_mov_b32_e32 v1, 15
+; GCN-NEXT:    scratch_store_dword v0, v1, off offset:4 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_sub_u32_e32 v0, 4, v0
+; GCN-NEXT:    scratch_load_dword v0, v0, off offset:124 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_endpgm
 bb:
   %i = alloca [32 x float], align 4, addrspace(5)
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i2 = tail call i32 @llvm.amdgcn.workitem.id.x()
   %i3 = zext i32 %i2 to i64
   %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i2
   %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
   store volatile i32 15, i32 addrspace(5)* %i8, align 4

 ; GFX9-PAL-NEXT:    v_and_b32_e32 v0, 15, v0
 ; GFX9-PAL-NEXT:    scratch_store_dword v2, v3, off
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GFX940-LABEL: store_load_vindex_foo:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 15
+; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX940-NEXT:    scratch_store_dword v1, v2, s32 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    scratch_load_dword v0, v0, s32 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX10-PAL-LABEL: store_load_vindex_foo:
 ; GFX10-PAL:       ; %bb.0: ; %bb
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, s32
 ; GFX10-PAL-NEXT:    v_and_b32_e32 v2, 15, v0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, 15
 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v1, v2, 2, v1
 ; GFX10-PAL-NEXT:    scratch_store_dword v0, v3, off
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_load_dword v0, v1, off glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: store_load_vindex_foo:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, 15
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; GCN-NEXT:    v_and_b32_e32 v0, v0, v2
+; GCN-NEXT:    scratch_store_dword v1, v2, s32 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-NEXT:    scratch_load_dword v0, v0, s32 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %i = alloca [32 x float], align 4, addrspace(5)
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
   %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*
   store volatile i32 15, i32 addrspace(5)* %i8, align 4
   %i9 = and i32 %idx, 15
   %i10 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %i9

 ; GFX9-PAL-LABEL: private_ptr_foo:
 ; GFX9-PAL:       ; %bb.0:
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 0x41200000
 ; GFX9-PAL-NEXT:    scratch_store_dword v0, v1, off offset:4
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GFX940-LABEL: private_ptr_foo:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0x41200000
+; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:4
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX10-PAL-LABEL: private_ptr_foo:
 ; GFX10-PAL:       ; %bb.0:
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 0x41200000
 ; GFX10-PAL-NEXT:    scratch_store_dword v0, v1, off offset:4
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: private_ptr_foo:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v1, 0x41200000
+; GCN-NEXT:    scratch_store_dword v0, v1, off offset:4
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr inbounds float, float addrspace(5)* %arg, i32 1
   store float 1.000000e+01, float addrspace(5)* %gep, align 4
   ret void
 }
 
 define amdgpu_kernel void @zero_init_small_offset_kernel() {
 ; GFX9-LABEL: zero_init_small_offset_kernel:
 ; GFX9:       ; %bb.0:

 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:288
 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:304
 ; GFX9-PAL-NEXT:    s_mov_b32 vcc_hi, 0
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:320
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: zero_init_small_offset_kernel:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_mov_b32 s0, 0
+; GFX940-NEXT:    s_mov_b32 s1, s0
+; GFX940-NEXT:    s_mov_b32 s2, s0
+; GFX940-NEXT:    s_mov_b32 s3, s0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:272
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:288
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:304
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:320
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX1010-PAL-LABEL: zero_init_small_offset_kernel:
 ; GFX1010-PAL:       ; %bb.0:
 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1

 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:256
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:272
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:288
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:304
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GFX940-LABEL: zero_init_small_offset_foo:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    scratch_load_dword v0, off, s32 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_mov_b32 s0, 0
+; GFX940-NEXT:    s_mov_b32 s1, s0
+; GFX940-NEXT:    s_mov_b32 s2, s0
+; GFX940-NEXT:    s_mov_b32 s3, s0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:256
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:272
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:288
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:304
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX10-PAL-LABEL: zero_init_small_offset_foo:
 ; GFX10-PAL:       ; %bb.0:
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_load_dword v0, off, s32 glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    s_mov_b32 s0, 0
 ; GFX10-PAL-NEXT:    s_mov_b32 s1, s0
 ; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX10-PAL-NEXT:    s_mov_b32 s3, s0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:256
 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:272
 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:288
 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:304
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: zero_init_small_offset_foo:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    scratch_load_dword v0, off, s32 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, 0
+; GCN-NEXT:    s_mov_b32 s1, s0
+; GCN-NEXT:    s_mov_b32 s2, s0
+; GCN-NEXT:    s_mov_b32 s3, s0
+; GCN-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GCN-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:256
+; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:272
+; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:288
+; GCN-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:304
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %padding = alloca [64 x i32], align 4, addrspace(5)
   %alloca = alloca [32 x i16], align 2, addrspace(5)
   %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
   %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
   %cast = bitcast [32 x i16] addrspace(5)* %alloca to i8 addrspace(5)*
   call void @llvm.memset.p5i8.i64(i8 addrspace(5)* align 2 dereferenceable(64) %cast, i8 0, i64 64, i1 false)
   ret void
 }

 ; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_addk_i32 s0, 0x104
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: store_load_sindex_small_offset_kernel:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 15
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
+; GFX940-NEXT:    s_and_b32 s0, s0, 15
+; GFX940-NEXT:    s_addk_i32 s1, 0x104
+; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
+; GFX940-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_addk_i32 s0, 0x104
+; GFX940-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX1010-PAL-LABEL: store_load_sindex_small_offset_kernel:
 ; GFX1010-PAL:       ; %bb.0: ; %bb
 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[4:5]
 ; GFX1010-PAL-NEXT:    s_mov_b32 s4, s0
 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1010-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
 ; GFX1010-PAL-NEXT:    s_add_u32 s4, s4, s3

 ; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_addk_i32 s0, 0x104
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: store_load_sindex_small_offset_foo:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
+; GFX940-NEXT:    s_and_b32 s0, s0, 15
+; GFX940-NEXT:    s_addk_i32 s1, 0x104
+; GFX940-NEXT:    v_mov_b32_e32 v0, 15
+; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
+; GFX940-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_addk_i32 s0, 0x104
+; GFX940-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX1010-PAL-LABEL: store_load_sindex_small_offset_foo:
 ; GFX1010-PAL:       ; %bb.0: ; %bb
 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1

 ; GFX9-PAL-NEXT:    v_add_u32_e32 v2, v1, v0
 ; GFX9-PAL-NEXT:    scratch_store_dword v2, v3, off
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    v_sub_u32_e32 v0, v1, v0
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: store_load_vindex_small_offset_kernel:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    scratch_load_dword v1, off, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v1, 15
+; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:260 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_sub_u32_e32 v0, 0x104, v0
+; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:124 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX1010-PAL-LABEL: store_load_vindex_small_offset_kernel:
 ; GFX1010-PAL:       ; %bb.0: ; %bb
 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1

 ; GFX9-PAL-NEXT:    v_and_b32_e32 v0, 15, v0
 ; GFX9-PAL-NEXT:    scratch_store_dword v2, v3, off
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GFX940-LABEL: store_load_vindex_small_offset_foo:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    scratch_load_dword v1, off, s32 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 15
+; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX940-NEXT:    scratch_store_dword v1, v2, s32 offset:256 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    scratch_load_dword v0, v0, s32 offset:256 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX10-PAL-LABEL: store_load_vindex_small_offset_foo:
 ; GFX10-PAL:       ; %bb.0: ; %bb
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x100
 ; GFX10-PAL-NEXT:    v_and_b32_e32 v2, 15, v0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, vcc_lo
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, 15
 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v1, v2, 2, v1
 ; GFX10-PAL-NEXT:    scratch_load_dword v2, off, s32 glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    scratch_store_dword v0, v3, off
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_load_dword v0, v1, off glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: store_load_vindex_small_offset_foo:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    scratch_load_dword v1, off, s32 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, 15
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; GCN-NEXT:    v_and_b32_e32 v0, v0, v2
+; GCN-NEXT:    scratch_store_dword v1, v2, s32 offset:256 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-NEXT:    scratch_load_dword v0, v0, s32 offset:256 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %padding = alloca [64 x i32], align 4, addrspace(5)
   %i = alloca [32 x float], align 4, addrspace(5)
   %pad_gep = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %padding, i32 0, i32 undef
   %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
   %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*

 ; GFX9-PAL-NEXT:    s_movk_i32 vcc_hi, 0x4010
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
 ; GFX9-PAL-NEXT:    s_movk_i32 vcc_hi, 0x4010
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
 ; GFX9-PAL-NEXT:    s_movk_i32 vcc_hi, 0x4010
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: zero_init_large_offset_kernel:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    scratch_load_dword v0, off, off offset:16 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_mov_b32 s0, 0
+; GFX940-NEXT:    s_mov_b32 s1, s0
+; GFX940-NEXT:    s_mov_b32 s2, s0
+; GFX940-NEXT:    s_mov_b32 s3, s0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4010
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi
+; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4010
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
+; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4010
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
+; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4010
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX1010-PAL-LABEL: zero_init_large_offset_kernel:
 ; GFX1010-PAL:       ; %bb.0:
 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1

 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
 ; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
 ; GFX9-PAL-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GFX940-LABEL: zero_init_large_offset_foo:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    scratch_load_dword v0, off, s32 offset:16 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_mov_b32 s0, 0
+; GFX940-NEXT:    s_mov_b32 s1, s0
+; GFX940-NEXT:    s_mov_b32 s2, s0
+; GFX940-NEXT:    s_mov_b32 s3, s0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi
+; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:16
+; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:32
+; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4010
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_hi offset:48
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1010-PAL-LABEL: zero_init_large_offset_foo:
 ; GFX1010-PAL:       ; %bb.0:
 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1010-PAL-NEXT:    scratch_load_dword v0, off, s32 offset:16 glc dlc
 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1010-PAL-NEXT:    s_mov_b32 s0, 0
 ; GFX1010-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4010

 ; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_addk_i32 s0, 0x4004
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: store_load_sindex_large_offset_kernel:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 15
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
+; GFX940-NEXT:    s_and_b32 s0, s0, 15
+; GFX940-NEXT:    s_addk_i32 s1, 0x4004
+; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
+; GFX940-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_addk_i32 s0, 0x4004
+; GFX940-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX1010-PAL-LABEL: store_load_sindex_large_offset_kernel:
 ; GFX1010-PAL:       ; %bb.0: ; %bb
 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[4:5]
 ; GFX1010-PAL-NEXT:    s_mov_b32 s4, s0
 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1010-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
 ; GFX1010-PAL-NEXT:    s_add_u32 s4, s4, s3

 ; GFX9-PAL-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s1
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_addk_i32 s0, 0x4004
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: store_load_sindex_large_offset_foo:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    scratch_load_dword v0, off, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_lshl_b32 s1, s0, 2
+; GFX940-NEXT:    s_and_b32 s0, s0, 15
+; GFX940-NEXT:    s_addk_i32 s1, 0x4004
+; GFX940-NEXT:    v_mov_b32_e32 v0, 15
+; GFX940-NEXT:    s_lshl_b32 s0, s0, 2
+; GFX940-NEXT:    scratch_store_dword off, v0, s1 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_addk_i32 s0, 0x4004
+; GFX940-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX1010-PAL-LABEL: store_load_sindex_large_offset_foo:
 ; GFX1010-PAL:       ; %bb.0: ; %bb
 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1

 ; GFX9-PAL-NEXT:    v_add_u32_e32 v2, v1, v0
 ; GFX9-PAL-NEXT:    scratch_store_dword v2, v3, off
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    v_sub_u32_e32 v0, v1, v0
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: store_load_vindex_large_offset_kernel:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    scratch_load_dword v1, off, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v1, 15
+; GFX940-NEXT:    s_movk_i32 vcc_hi, 0x4004
+; GFX940-NEXT:    scratch_store_dword v0, v1, vcc_hi sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_sub_u32_e32 v0, 0x4004, v0
+; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:124 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX1010-PAL-LABEL: store_load_vindex_large_offset_kernel:
 ; GFX1010-PAL:       ; %bb.0: ; %bb
 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1

 ; GFX9-PAL-NEXT:    v_and_b32_e32 v0, 15, v0
 ; GFX9-PAL-NEXT:    scratch_store_dword v2, v3, off
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GFX940-LABEL: store_load_vindex_large_offset_foo:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    scratch_load_dword v1, off, s32 offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 15
+; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
+; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX940-NEXT:    scratch_store_dword v1, v2, vcc_hi sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    s_add_i32 vcc_hi, s32, 0x4004
+; GFX940-NEXT:    scratch_load_dword v0, v0, vcc_hi sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX10-PAL-LABEL: store_load_vindex_large_offset_foo:
 ; GFX10-PAL:       ; %bb.0: ; %bb
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    s_add_i32 vcc_lo, s32, 0x4004
 ; GFX10-PAL-NEXT:    v_and_b32_e32 v2, 15, v0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, vcc_lo
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, 15
 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v1, v2, 2, v1
 ; GFX10-PAL-NEXT:    scratch_load_dword v2, off, s32 offset:4 glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    scratch_store_dword v0, v3, off
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_load_dword v0, v1, off glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: store_load_vindex_large_offset_foo:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    scratch_load_dword v1, off, s32 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, 15
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; GCN-NEXT:    v_and_b32_e32 v0, v0, v2
+; GCN-NEXT:    s_add_u32 vcc_hi, s32, 0x4000
+; GCN-NEXT:    scratch_store_dword v1, v2, vcc_hi sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-NEXT:    s_add_u32 vcc_hi, s32, 0x4000
+; GCN-NEXT:    scratch_load_dword v0, v0, vcc_hi sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %padding = alloca [4096 x i32], align 4, addrspace(5)
   %i = alloca [32 x float], align 4, addrspace(5)
   %pad_gep = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %padding, i32 0, i32 undef
   %pad_load = load volatile i32, i32 addrspace(5)* %pad_gep, align 4
   %i1 = bitcast [32 x float] addrspace(5)* %i to i8 addrspace(5)*
   %i7 = getelementptr inbounds [32 x float], [32 x float] addrspace(5)* %i, i32 0, i32 %idx
   %i8 = bitcast float addrspace(5)* %i7 to i32 addrspace(5)*

 ; GFX9-PAL-NEXT:    s_add_i32 s0, s0, 4
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s0 offset:3712
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 offset:3712 glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: store_load_large_imm_offset_kernel:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    v_mov_b32_e32 v0, 13
+; GFX940-NEXT:    scratch_store_dword off, v0, off offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0x3000
+; GFX940-NEXT:    v_mov_b32_e32 v1, 15
+; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:3716 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:3716 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX1010-PAL-LABEL: store_load_large_imm_offset_kernel:
 ; GFX1010-PAL:       ; %bb.0: ; %bb
 ; GFX1010-PAL-NEXT:    s_getpc_b64 s[2:3]
 ; GFX1010-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX1010-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX1010-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1010-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX1010-PAL-NEXT:    s_add_u32 s2, s2, s1

 ; GFX9-PAL-NEXT:    s_add_i32 s0, s0, vcc_hi
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s0 offset:3712
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, off, s0 offset:3712 glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GFX940-LABEL: store_load_large_imm_offset_foo:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 13
+; GFX940-NEXT:    scratch_store_dword off, v0, s32 offset:4 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0x3000
+; GFX940-NEXT:    v_mov_b32_e32 v1, 15
+; GFX940-NEXT:    scratch_store_dword v0, v1, s32 offset:3716 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dword v0, v0, s32 offset:3716 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX10-PAL-LABEL: store_load_large_imm_offset_foo:
 ; GFX10-PAL:       ; %bb.0: ; %bb
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 13
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-PAL-NEXT:    s_movk_i32 s0, 0x3800
 ; GFX10-PAL-NEXT:    s_add_i32 vcc_lo, s32, 4
 ; GFX10-PAL-NEXT:    s_add_i32 s0, s0, vcc_lo
 ; GFX10-PAL-NEXT:    scratch_store_dword off, v0, s32 offset:4
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_store_dword off, v1, s0 offset:1664
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_load_dword v0, off, s0 offset:1664 glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: store_load_large_imm_offset_foo:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, 13
+; GCN-NEXT:    scratch_store_dword off, v0, s32 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, 0x3000
+; GCN-NEXT:    v_mov_b32_e32 v1, 15
+; GCN-NEXT:    scratch_store_dword v0, v1, s32 offset:3712 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    scratch_load_dword v0, v0, s32 offset:3712 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %i = alloca [4096 x i32], align 4, addrspace(5)
   %i1 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 undef
   store volatile i32 13, i32 addrspace(5)* %i1, align 4
   %i7 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
   store volatile i32 15, i32 addrspace(5)* %i7, align 4
   %i10 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000
   %i12 = load volatile i32, i32 addrspace(5)* %i10, align 4

 ; GFX9-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v1
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX9-PAL-NEXT:    scratch_store_dword v0, v1, off offset:1024
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: store_load_vidx_sidx_offset:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v1, 15
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
+; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:1028 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:1028 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX10-PAL-LABEL: store_load_vidx_sidx_offset:
 ; GFX10-PAL:       ; %bb.0: ; %bb
 ; GFX10-PAL-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-PAL-NEXT:    s_mov_b32 s4, s0
 ; GFX10-PAL-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_and_b32 s5, s5, 0xffff
 ; GFX10-PAL-NEXT:    s_add_u32 s4, s4, s3
 ; GFX10-PAL-NEXT:    s_addc_u32 s5, s5, 0
 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
 ; GFX10-PAL-NEXT:    s_load_dword s0, s[0:1], 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-PAL-NEXT:    v_add_nc_u32_e32 v0, s0, v0
 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, 4
 ; GFX10-PAL-NEXT:    scratch_store_dword v0, v1, off offset:1024
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    s_endpgm
+; GCN-LABEL: store_load_vidx_sidx_offset:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v1, 15
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
+; GCN-NEXT:    scratch_store_dword v0, v1, off offset:1028 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    scratch_load_dword v0, v0, off offset:1028 sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_endpgm
 bb:
   %alloca = alloca [32 x i32], align 4, addrspace(5)
   %vidx = tail call i32 @llvm.amdgcn.workitem.id.x()
   %add1 = add nsw i32 %sidx, %vidx
   %add2 = add nsw i32 %add1, 256
   %gep = getelementptr inbounds [32 x i32], [32 x i32] addrspace(5)* %alloca, i32 0, i32 %add2
   store volatile i32 15, i32 addrspace(5)* %gep, align 4
   %load = load volatile i32, i32 addrspace(5)* %gep, align 4

 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-PAL-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GFX940-LABEL: store_load_i64_aligned:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v2, 15
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX10-PAL-LABEL: store_load_i64_aligned:
 ; GFX10-PAL:       ; %bb.0: ; %bb
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-PAL-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: store_load_i64_aligned:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, 15
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   store volatile i64 15, i64 addrspace(5)* %arg, align 8
   %load = load volatile i64, i64 addrspace(5)* %arg, align 8
   ret void
 }
 
 define void @store_load_i64_unaligned(i64 addrspace(5)* nocapture %arg) {
 ; GFX9-LABEL: store_load_i64_unaligned:

 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-PAL-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GFX940-LABEL: store_load_i64_unaligned:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v2, 15
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX10-PAL-LABEL: store_load_i64_unaligned:
 ; GFX10-PAL:       ; %bb.0: ; %bb
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-PAL-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: store_load_i64_unaligned:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, 15
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   store volatile i64 15, i64 addrspace(5)* %arg, align 1
   %load = load volatile i64, i64 addrspace(5)* %arg, align 1
   ret void
 }
 
 define void @store_load_v3i32_unaligned(<3 x i32> addrspace(5)* nocapture %arg) {
 ; GFX9-LABEL: store_load_v3i32_unaligned:

 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 2
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, 3
 ; GFX9-PAL-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GFX940-LABEL: store_load_v3i32_unaligned:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v2, 1
+; GFX940-NEXT:    v_mov_b32_e32 v3, 2
+; GFX940-NEXT:    v_mov_b32_e32 v4, 3
+; GFX940-NEXT:    scratch_store_dwordx3 v0, v[2:4], off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dwordx3 v[0:2], v0, off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX10-PAL-LABEL: store_load_v3i32_unaligned:
 ; GFX10-PAL:       ; %bb.0: ; %bb
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 2
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, 3
 ; GFX10-PAL-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: store_load_v3i32_unaligned:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, 1
+; GCN-NEXT:    v_mov_b32_e32 v3, 2
+; GCN-NEXT:    v_mov_b32_e32 v4, 3
+; GCN-NEXT:    scratch_store_dwordx3 v0, v[2:4], off sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    scratch_load_dwordx3 v[0:2], v0, off sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   store volatile <3 x i32> <i32 1, i32 2, i32 3>, <3 x i32> addrspace(5)* %arg, align 1
   %load = load volatile <3 x i32>, <3 x i32> addrspace(5)* %arg, align 1
   ret void
 }
 
 define void @store_load_v4i32_unaligned(<4 x i32> addrspace(5)* nocapture %arg) {
 ; GFX9-LABEL: store_load_v4i32_unaligned:

 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v3, 3
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v4, 4
 ; GFX9-PAL-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GFX940-LABEL: store_load_v4i32_unaligned:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v2, 1
+; GFX940-NEXT:    v_mov_b32_e32 v3, 2
+; GFX940-NEXT:    v_mov_b32_e32 v4, 3
+; GFX940-NEXT:    v_mov_b32_e32 v5, 4
+; GFX940-NEXT:    scratch_store_dwordx4 v0, v[2:5], off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dwordx4 v[0:3], v0, off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX10-PAL-LABEL: store_load_v4i32_unaligned:
 ; GFX10-PAL:       ; %bb.0: ; %bb
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 2
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, 3
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v4, 4
 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc dlc
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: store_load_v4i32_unaligned:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, 1
+; GCN-NEXT:    v_mov_b32_e32 v3, 2
+; GCN-NEXT:    v_mov_b32_e32 v4, 3
+; GCN-NEXT:    v_mov_b32_e32 v5, 4
+; GCN-NEXT:    scratch_store_dwordx4 v0, v[2:5], off sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    scratch_load_dwordx4 v[0:3], v0, off sc0 sc1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   store volatile <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> addrspace(5)* %arg, align 1
   %load = load volatile <4 x i32>, <4 x i32> addrspace(5)* %arg, align 1
   ret void
 }
 
 define void @store_load_i32_negative_unaligned(i8 addrspace(5)* nocapture %arg) {
 ; GFX9-LABEL: store_load_i32_negative_unaligned:

 ; GFX9-PAL-NEXT:    v_add_u32_e32 v0, -1, v0
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX9-PAL-NEXT:    scratch_store_byte v0, v1, off
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    scratch_load_ubyte v0, v0, off glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GFX940-LABEL: store_load_i32_negative_unaligned:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_add_u32_e32 v0, -1, v0
+; GFX940-NEXT:    v_mov_b32_e32 v1, 1
+; GFX940-NEXT:    scratch_store_byte v0, v1, off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_ubyte v0, v0, off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1010-PAL-LABEL: store_load_i32_negative_unaligned:
 ; GFX1010-PAL:       ; %bb.0: ; %bb
 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1010-PAL-NEXT:    v_add_nc_u32_e32 v0, -1, v0
 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX1010-PAL-NEXT:    scratch_store_byte v0, v1, off
 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0

 ; GFX9-PAL-NEXT:    v_add_u32_e32 v0, 0xffffef7f, v0
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX9-PAL-NEXT:    scratch_store_byte v0, v1, off
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    scratch_load_ubyte v0, v0, off glc
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-PAL-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GFX940-LABEL: store_load_i32_large_negative_unaligned:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    s_movk_i32 s0, 0xef7f
+; GFX940-NEXT:    v_mov_b32_e32 v1, 1
+; GFX940-NEXT:    scratch_store_byte v0, v1, s0 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_ubyte v0, v0, s0 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX1010-PAL-LABEL: store_load_i32_large_negative_unaligned:
 ; GFX1010-PAL:       ; %bb.0: ; %bb
 ; GFX1010-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1010-PAL-NEXT:    v_add_nc_u32_e32 v0, 0xffffefff, v0
 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX1010-PAL-NEXT:    scratch_store_byte v0, v1, off offset:-128
 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0

 ; GFX9-PAL-NEXT:    ; use v0
 ; GFX9-PAL-NEXT:    ;;#ASMEND
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 0x810
 ; GFX9-PAL-NEXT:    ;;#ASMSTART
 ; GFX9-PAL-NEXT:    ; use v0
 ; GFX9-PAL-NEXT:    ;;#ASMEND
 ; GFX9-PAL-NEXT:    s_endpgm
 ;
+; GFX940-LABEL: large_offset:
+; GFX940:       ; %bb.0: ; %bb
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v3, v0
+; GFX940-NEXT:    scratch_store_dwordx4 off, v[0:3], off offset:3024 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    scratch_load_dwordx4 v[0:3], off, off offset:3024 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 16
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; use v0
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0x810
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; use v0
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_endpgm
+;
 ; GFX10-PAL-LABEL: large_offset:
 ; GFX10-PAL:       ; %bb.0: ; %bb
 ; GFX10-PAL-NEXT:    s_getpc_b64 s[2:3]
 ; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX10-PAL-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX10-PAL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX10-PAL-NEXT:    s_add_u32 s2, s2, s0

llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll

 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v1, v0, s[0:1]
 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
 ;
 ; GFX940-NOTTGSPLIT-LABEL: private_nontemporal_load_1:
 ; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
 ; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
 ; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
+; GFX940-NOTTGSPLIT-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NOTTGSPLIT-NEXT:    v_lshl_add_u32 v0, v0, 2, s4
-; GFX940-NOTTGSPLIT-NEXT:    scratch_load_dword v0, v0, off nt
+; GFX940-NOTTGSPLIT-NEXT:    scratch_load_dword v0, v0, s4 nt
 ; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
 ;
 ; GFX940-TGSPLIT-LABEL: private_nontemporal_load_1:
 ; GFX940-TGSPLIT:       ; %bb.0: ; %entry
 ; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
 ; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
+; GFX940-TGSPLIT-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-TGSPLIT-NEXT:    v_lshl_add_u32 v0, v0, 2, s4
-; GFX940-TGSPLIT-NEXT:    scratch_load_dword v0, v0, off nt
+; GFX940-TGSPLIT-NEXT:    scratch_load_dword v0, v0, s4 nt
 ; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-TGSPLIT-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX940-TGSPLIT-NEXT:    s_endpgm
     i32 addrspace(5)* %in, i32 addrspace(1)* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val.gep = getelementptr inbounds i32, i32 addrspace(5)* %in, i32 %tid
   %val = load i32, i32 addrspace(5)* %val.gep, align 4, !nontemporal !0

 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90A-TGSPLIT-NEXT:    buffer_store_dword v1, v0, s[8:11], 0 offen glc slc
 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
 ;
 ; GFX940-NOTTGSPLIT-LABEL: private_nontemporal_store_1:
 ; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
 ; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
 ; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
+; GFX940-NOTTGSPLIT-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NOTTGSPLIT-NEXT:    v_lshl_add_u32 v0, v0, 2, s4
 ; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s0, s[2:3], 0x0
 ; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-NOTTGSPLIT-NEXT:    scratch_store_dword v0, v1, off nt
+; GFX940-NOTTGSPLIT-NEXT:    scratch_store_dword v0, v1, s4 nt
 ; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
 ;
 ; GFX940-TGSPLIT-LABEL: private_nontemporal_store_1:
 ; GFX940-TGSPLIT:       ; %bb.0: ; %entry
 ; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
 ; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
+; GFX940-TGSPLIT-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-TGSPLIT-NEXT:    v_lshl_add_u32 v0, v0, 2, s4
 ; GFX940-TGSPLIT-NEXT:    s_load_dword s0, s[2:3], 0x0
 ; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-TGSPLIT-NEXT:    scratch_store_dword v0, v1, off nt
+; GFX940-TGSPLIT-NEXT:    scratch_store_dword v0, v1, s4 nt
 ; GFX940-TGSPLIT-NEXT:    s_endpgm
     i32 addrspace(1)* %in, i32 addrspace(5)* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val = load i32, i32 addrspace(1)* %in, align 4
   %out.gep = getelementptr inbounds i32, i32 addrspace(5)* %out, i32 %tid
   store i32 %val, i32 addrspace(5)* %out.gep, !nontemporal !0
   ret void
 }
 
 !0 = !{i32 1}
 declare i32 @llvm.amdgcn.workitem.id.x()

llvm/test/MC/AMDGPU/flat-scratch-gfx940.s

+// RUN: llvm-mc -arch=amdgcn -mcpu=gfx940 -show-encoding %s | FileCheck -check-prefix=GFX940 %s
+
+scratch_load_dword a2, v4, s6
+// GFX940: scratch_load_dword a2, v4, s6           ; encoding: [0x00,0x60,0x50,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_dword a2, v4, s6 offset:16
+// GFX940: scratch_load_dword a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x50,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_dword a2, v4, off
+// GFX940: scratch_load_dword a2, v4, off          ; encoding: [0x00,0x60,0x50,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_dword a2, v4, off offset:16
+// GFX940: scratch_load_dword a2, v4, off offset:16 ; encoding: [0x10,0x60,0x50,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_dword a2, off, s6
+// GFX940: scratch_load_dword a2, off, s6          ; encoding: [0x00,0x40,0x50,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_dword a2, off, s6 offset:16
+// GFX940: scratch_load_dword a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x50,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_dword a2, off, off
+// GFX940: scratch_load_dword a2, off, off         ; encoding: [0x00,0x40,0x50,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_dword a2, off, off offset:16
+// GFX940: scratch_load_dword a2, off, off offset:16 ; encoding: [0x10,0x40,0x50,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_dword v2, v4, s6
+// GFX940: scratch_load_dword v2, v4, s6           ; encoding: [0x00,0x60,0x50,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_dword v2, v4, s6 offset:16
+// GFX940: scratch_load_dword v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x50,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_dword v2, v4, off
+// GFX940: scratch_load_dword v2, v4, off          ; encoding: [0x00,0x60,0x50,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_dword v2, v4, off offset:16
+// GFX940: scratch_load_dword v2, v4, off offset:16 ; encoding: [0x10,0x60,0x50,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_dword v2, off, s6
+// GFX940: scratch_load_dword v2, off, s6          ; encoding: [0x00,0x40,0x50,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_dword v2, off, s6 offset:16
+// GFX940: scratch_load_dword v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x50,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_dword v2, off, off
+// GFX940: scratch_load_dword v2, off, off         ; encoding: [0x00,0x40,0x50,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_dword v2, off, off offset:16
+// GFX940: scratch_load_dword v2, off, off offset:16 ; encoding: [0x10,0x40,0x50,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_dwordx2 a[2:3], v4, s6
+// GFX940: scratch_load_dwordx2 a[2:3], v4, s6     ; encoding: [0x00,0x60,0x54,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_dwordx2 a[2:3], v4, s6 offset:16
+// GFX940: scratch_load_dwordx2 a[2:3], v4, s6 offset:16 ; encoding: [0x10,0x60,0x54,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_dwordx2 a[2:3], v4, off
+// GFX940: scratch_load_dwordx2 a[2:3], v4, off    ; encoding: [0x00,0x60,0x54,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_dwordx2 a[2:3], v4, off offset:16
+// GFX940: scratch_load_dwordx2 a[2:3], v4, off offset:16 ; encoding: [0x10,0x60,0x54,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_dwordx2 a[2:3], off, s6
+// GFX940: scratch_load_dwordx2 a[2:3], off, s6    ; encoding: [0x00,0x40,0x54,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_dwordx2 a[2:3], off, s6 offset:16
+// GFX940: scratch_load_dwordx2 a[2:3], off, s6 offset:16 ; encoding: [0x10,0x40,0x54,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_dwordx2 a[2:3], off, off
+// GFX940: scratch_load_dwordx2 a[2:3], off, off   ; encoding: [0x00,0x40,0x54,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_dwordx2 a[2:3], off, off offset:16
+// GFX940: scratch_load_dwordx2 a[2:3], off, off offset:16 ; encoding: [0x10,0x40,0x54,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_dwordx2 v[2:3], v4, s6
+// GFX940: scratch_load_dwordx2 v[2:3], v4, s6     ; encoding: [0x00,0x60,0x54,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_dwordx2 v[2:3], v4, s6 offset:16
+// GFX940: scratch_load_dwordx2 v[2:3], v4, s6 offset:16 ; encoding: [0x10,0x60,0x54,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_dwordx2 v[2:3], v4, off
+// GFX940: scratch_load_dwordx2 v[2:3], v4, off    ; encoding: [0x00,0x60,0x54,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_dwordx2 v[2:3], v4, off offset:16
+// GFX940: scratch_load_dwordx2 v[2:3], v4, off offset:16 ; encoding: [0x10,0x60,0x54,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_dwordx2 v[2:3], off, s6
+// GFX940: scratch_load_dwordx2 v[2:3], off, s6    ; encoding: [0x00,0x40,0x54,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_dwordx2 v[2:3], off, s6 offset:16
+// GFX940: scratch_load_dwordx2 v[2:3], off, s6 offset:16 ; encoding: [0x10,0x40,0x54,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_dwordx2 v[2:3], off, off
+// GFX940: scratch_load_dwordx2 v[2:3], off, off   ; encoding: [0x00,0x40,0x54,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_dwordx2 v[2:3], off, off offset:16
+// GFX940: scratch_load_dwordx2 v[2:3], off, off offset:16 ; encoding: [0x10,0x40,0x54,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_dwordx3 a[2:4], v4, s6
+// GFX940: scratch_load_dwordx3 a[2:4], v4, s6     ; encoding: [0x00,0x60,0x58,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_dwordx3 a[2:4], v4, s6 offset:16
+// GFX940: scratch_load_dwordx3 a[2:4], v4, s6 offset:16 ; encoding: [0x10,0x60,0x58,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_dwordx3 a[2:4], v4, off
+// GFX940: scratch_load_dwordx3 a[2:4], v4, off    ; encoding: [0x00,0x60,0x58,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_dwordx3 a[2:4], v4, off offset:16
+// GFX940: scratch_load_dwordx3 a[2:4], v4, off offset:16 ; encoding: [0x10,0x60,0x58,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_dwordx3 a[2:4], off, s6
+// GFX940: scratch_load_dwordx3 a[2:4], off, s6    ; encoding: [0x00,0x40,0x58,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_dwordx3 a[2:4], off, s6 offset:16
+// GFX940: scratch_load_dwordx3 a[2:4], off, s6 offset:16 ; encoding: [0x10,0x40,0x58,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_dwordx3 a[2:4], off, off
+// GFX940: scratch_load_dwordx3 a[2:4], off, off   ; encoding: [0x00,0x40,0x58,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_dwordx3 a[2:4], off, off offset:16
+// GFX940: scratch_load_dwordx3 a[2:4], off, off offset:16 ; encoding: [0x10,0x40,0x58,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_dwordx3 v[2:4], v4, s6
+// GFX940: scratch_load_dwordx3 v[2:4], v4, s6     ; encoding: [0x00,0x60,0x58,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_dwordx3 v[2:4], v4, s6 offset:16
+// GFX940: scratch_load_dwordx3 v[2:4], v4, s6 offset:16 ; encoding: [0x10,0x60,0x58,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_dwordx3 v[2:4], v4, off
+// GFX940: scratch_load_dwordx3 v[2:4], v4, off    ; encoding: [0x00,0x60,0x58,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_dwordx3 v[2:4], v4, off offset:16
+// GFX940: scratch_load_dwordx3 v[2:4], v4, off offset:16 ; encoding: [0x10,0x60,0x58,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_dwordx3 v[2:4], off, s6
+// GFX940: scratch_load_dwordx3 v[2:4], off, s6    ; encoding: [0x00,0x40,0x58,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_dwordx3 v[2:4], off, s6 offset:16
+// GFX940: scratch_load_dwordx3 v[2:4], off, s6 offset:16 ; encoding: [0x10,0x40,0x58,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_dwordx3 v[2:4], off, off
+// GFX940: scratch_load_dwordx3 v[2:4], off, off   ; encoding: [0x00,0x40,0x58,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_dwordx3 v[2:4], off, off offset:16
+// GFX940: scratch_load_dwordx3 v[2:4], off, off offset:16 ; encoding: [0x10,0x40,0x58,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_dwordx4 a[2:5], v4, s6
+// GFX940: scratch_load_dwordx4 a[2:5], v4, s6     ; encoding: [0x00,0x60,0x5c,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_dwordx4 a[2:5], v4, s6 offset:16
+// GFX940: scratch_load_dwordx4 a[2:5], v4, s6 offset:16 ; encoding: [0x10,0x60,0x5c,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_dwordx4 a[2:5], v4, off
+// GFX940: scratch_load_dwordx4 a[2:5], v4, off    ; encoding: [0x00,0x60,0x5c,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_dwordx4 a[2:5], v4, off offset:16
+// GFX940: scratch_load_dwordx4 a[2:5], v4, off offset:16 ; encoding: [0x10,0x60,0x5c,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_dwordx4 a[2:5], off, s6
+// GFX940: scratch_load_dwordx4 a[2:5], off, s6    ; encoding: [0x00,0x40,0x5c,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_dwordx4 a[2:5], off, s6 offset:16
+// GFX940: scratch_load_dwordx4 a[2:5], off, s6 offset:16 ; encoding: [0x10,0x40,0x5c,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_dwordx4 a[2:5], off, off
+// GFX940: scratch_load_dwordx4 a[2:5], off, off   ; encoding: [0x00,0x40,0x5c,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_dwordx4 a[2:5], off, off offset:16
+// GFX940: scratch_load_dwordx4 a[2:5], off, off offset:16 ; encoding: [0x10,0x40,0x5c,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_dwordx4 v[2:5], v4, s6
+// GFX940: scratch_load_dwordx4 v[2:5], v4, s6     ; encoding: [0x00,0x60,0x5c,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_dwordx4 v[2:5], v4, s6 offset:16
+// GFX940: scratch_load_dwordx4 v[2:5], v4, s6 offset:16 ; encoding: [0x10,0x60,0x5c,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_dwordx4 v[2:5], v4, off
+// GFX940: scratch_load_dwordx4 v[2:5], v4, off    ; encoding: [0x00,0x60,0x5c,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_dwordx4 v[2:5], v4, off offset:16
+// GFX940: scratch_load_dwordx4 v[2:5], v4, off offset:16 ; encoding: [0x10,0x60,0x5c,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_dwordx4 v[2:5], off, s6
+// GFX940: scratch_load_dwordx4 v[2:5], off, s6    ; encoding: [0x00,0x40,0x5c,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_dwordx4 v[2:5], off, s6 offset:16
+// GFX940: scratch_load_dwordx4 v[2:5], off, s6 offset:16 ; encoding: [0x10,0x40,0x5c,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_dwordx4 v[2:5], off, off
+// GFX940: scratch_load_dwordx4 v[2:5], off, off   ; encoding: [0x00,0x40,0x5c,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_dwordx4 v[2:5], off, off offset:16
+// GFX940: scratch_load_dwordx4 v[2:5], off, off offset:16 ; encoding: [0x10,0x40,0x5c,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_sbyte a2, v4, s6
+// GFX940: scratch_load_sbyte a2, v4, s6           ; encoding: [0x00,0x60,0x44,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_sbyte a2, v4, s6 offset:16
+// GFX940: scratch_load_sbyte a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x44,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_sbyte a2, v4, off
+// GFX940: scratch_load_sbyte a2, v4, off          ; encoding: [0x00,0x60,0x44,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_sbyte a2, v4, off offset:16
+// GFX940: scratch_load_sbyte a2, v4, off offset:16 ; encoding: [0x10,0x60,0x44,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_sbyte a2, off, s6
+// GFX940: scratch_load_sbyte a2, off, s6          ; encoding: [0x00,0x40,0x44,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_sbyte a2, off, s6 offset:16
+// GFX940: scratch_load_sbyte a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x44,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_sbyte a2, off, off
+// GFX940: scratch_load_sbyte a2, off, off         ; encoding: [0x00,0x40,0x44,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_sbyte a2, off, off offset:16
+// GFX940: scratch_load_sbyte a2, off, off offset:16 ; encoding: [0x10,0x40,0x44,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_sbyte v2, v4, s6
+// GFX940: scratch_load_sbyte v2, v4, s6           ; encoding: [0x00,0x60,0x44,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_sbyte v2, v4, s6 offset:16
+// GFX940: scratch_load_sbyte v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x44,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_sbyte v2, v4, off
+// GFX940: scratch_load_sbyte v2, v4, off          ; encoding: [0x00,0x60,0x44,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_sbyte v2, v4, off offset:16
+// GFX940: scratch_load_sbyte v2, v4, off offset:16 ; encoding: [0x10,0x60,0x44,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_sbyte v2, off, s6
+// GFX940: scratch_load_sbyte v2, off, s6          ; encoding: [0x00,0x40,0x44,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_sbyte v2, off, s6 offset:16
+// GFX940: scratch_load_sbyte v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x44,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_sbyte v2, off, off
+// GFX940: scratch_load_sbyte v2, off, off         ; encoding: [0x00,0x40,0x44,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_sbyte v2, off, off offset:16
+// GFX940: scratch_load_sbyte v2, off, off offset:16 ; encoding: [0x10,0x40,0x44,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_sbyte_d16 a2, v4, s6
+// GFX940: scratch_load_sbyte_d16 a2, v4, s6       ; encoding: [0x00,0x60,0x88,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_sbyte_d16 a2, v4, s6 offset:16
+// GFX940: scratch_load_sbyte_d16 a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x88,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_sbyte_d16 a2, v4, off
+// GFX940: scratch_load_sbyte_d16 a2, v4, off      ; encoding: [0x00,0x60,0x88,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_sbyte_d16 a2, v4, off offset:16
+// GFX940: scratch_load_sbyte_d16 a2, v4, off offset:16 ; encoding: [0x10,0x60,0x88,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_sbyte_d16 a2, off, s6
+// GFX940: scratch_load_sbyte_d16 a2, off, s6      ; encoding: [0x00,0x40,0x88,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_sbyte_d16 a2, off, s6 offset:16
+// GFX940: scratch_load_sbyte_d16 a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x88,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_sbyte_d16 a2, off, off
+// GFX940: scratch_load_sbyte_d16 a2, off, off     ; encoding: [0x00,0x40,0x88,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_sbyte_d16 a2, off, off offset:16
+// GFX940: scratch_load_sbyte_d16 a2, off, off offset:16 ; encoding: [0x10,0x40,0x88,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_sbyte_d16 v2, v4, s6
+// GFX940: scratch_load_sbyte_d16 v2, v4, s6       ; encoding: [0x00,0x60,0x88,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_sbyte_d16 v2, v4, s6 offset:16
+// GFX940: scratch_load_sbyte_d16 v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x88,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_sbyte_d16 v2, v4, off
+// GFX940: scratch_load_sbyte_d16 v2, v4, off      ; encoding: [0x00,0x60,0x88,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_sbyte_d16 v2, v4, off offset:16
+// GFX940: scratch_load_sbyte_d16 v2, v4, off offset:16 ; encoding: [0x10,0x60,0x88,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_sbyte_d16 v2, off, s6
+// GFX940: scratch_load_sbyte_d16 v2, off, s6      ; encoding: [0x00,0x40,0x88,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_sbyte_d16 v2, off, s6 offset:16
+// GFX940: scratch_load_sbyte_d16 v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x88,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_sbyte_d16 v2, off, off
+// GFX940: scratch_load_sbyte_d16 v2, off, off     ; encoding: [0x00,0x40,0x88,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_sbyte_d16 v2, off, off offset:16
+// GFX940: scratch_load_sbyte_d16 v2, off, off offset:16 ; encoding: [0x10,0x40,0x88,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_sbyte_d16_hi a2, v4, s6
+// GFX940: scratch_load_sbyte_d16_hi a2, v4, s6    ; encoding: [0x00,0x60,0x8c,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_sbyte_d16_hi a2, v4, s6 offset:16
+// GFX940: scratch_load_sbyte_d16_hi a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x8c,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_sbyte_d16_hi a2, v4, off
+// GFX940: scratch_load_sbyte_d16_hi a2, v4, off   ; encoding: [0x00,0x60,0x8c,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_sbyte_d16_hi a2, v4, off offset:16
+// GFX940: scratch_load_sbyte_d16_hi a2, v4, off offset:16 ; encoding: [0x10,0x60,0x8c,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_sbyte_d16_hi a2, off, s6
+// GFX940: scratch_load_sbyte_d16_hi a2, off, s6   ; encoding: [0x00,0x40,0x8c,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_sbyte_d16_hi a2, off, s6 offset:16
+// GFX940: scratch_load_sbyte_d16_hi a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x8c,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_sbyte_d16_hi a2, off, off
+// GFX940: scratch_load_sbyte_d16_hi a2, off, off  ; encoding: [0x00,0x40,0x8c,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_sbyte_d16_hi a2, off, off offset:16
+// GFX940: scratch_load_sbyte_d16_hi a2, off, off offset:16 ; encoding: [0x10,0x40,0x8c,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_sbyte_d16_hi v2, v4, s6
+// GFX940: scratch_load_sbyte_d16_hi v2, v4, s6    ; encoding: [0x00,0x60,0x8c,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_sbyte_d16_hi v2, v4, s6 offset:16
+// GFX940: scratch_load_sbyte_d16_hi v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x8c,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_sbyte_d16_hi v2, v4, off
+// GFX940: scratch_load_sbyte_d16_hi v2, v4, off   ; encoding: [0x00,0x60,0x8c,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_sbyte_d16_hi v2, v4, off offset:16
+// GFX940: scratch_load_sbyte_d16_hi v2, v4, off offset:16 ; encoding: [0x10,0x60,0x8c,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_sbyte_d16_hi v2, off, s6
+// GFX940: scratch_load_sbyte_d16_hi v2, off, s6   ; encoding: [0x00,0x40,0x8c,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_sbyte_d16_hi v2, off, s6 offset:16
+// GFX940: scratch_load_sbyte_d16_hi v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x8c,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_sbyte_d16_hi v2, off, off
+// GFX940: scratch_load_sbyte_d16_hi v2, off, off  ; encoding: [0x00,0x40,0x8c,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_sbyte_d16_hi v2, off, off offset:16
+// GFX940: scratch_load_sbyte_d16_hi v2, off, off offset:16 ; encoding: [0x10,0x40,0x8c,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_short_d16 a2, v4, s6
+// GFX940: scratch_load_short_d16 a2, v4, s6       ; encoding: [0x00,0x60,0x90,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_short_d16 a2, v4, s6 offset:16
+// GFX940: scratch_load_short_d16 a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x90,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_short_d16 a2, v4, off
+// GFX940: scratch_load_short_d16 a2, v4, off      ; encoding: [0x00,0x60,0x90,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_short_d16 a2, v4, off offset:16
+// GFX940: scratch_load_short_d16 a2, v4, off offset:16 ; encoding: [0x10,0x60,0x90,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_short_d16 a2, off, s6
+// GFX940: scratch_load_short_d16 a2, off, s6      ; encoding: [0x00,0x40,0x90,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_short_d16 a2, off, s6 offset:16
+// GFX940: scratch_load_short_d16 a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x90,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_short_d16 a2, off, off
+// GFX940: scratch_load_short_d16 a2, off, off     ; encoding: [0x00,0x40,0x90,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_short_d16 a2, off, off offset:16
+// GFX940: scratch_load_short_d16 a2, off, off offset:16 ; encoding: [0x10,0x40,0x90,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_short_d16 v2, v4, s6
+// GFX940: scratch_load_short_d16 v2, v4, s6       ; encoding: [0x00,0x60,0x90,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_short_d16 v2, v4, s6 offset:16
+// GFX940: scratch_load_short_d16 v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x90,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_short_d16 v2, v4, off
+// GFX940: scratch_load_short_d16 v2, v4, off      ; encoding: [0x00,0x60,0x90,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_short_d16 v2, v4, off offset:16
+// GFX940: scratch_load_short_d16 v2, v4, off offset:16 ; encoding: [0x10,0x60,0x90,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_short_d16 v2, off, s6
+// GFX940: scratch_load_short_d16 v2, off, s6      ; encoding: [0x00,0x40,0x90,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_short_d16 v2, off, s6 offset:16
+// GFX940: scratch_load_short_d16 v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x90,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_short_d16 v2, off, off
+// GFX940: scratch_load_short_d16 v2, off, off     ; encoding: [0x00,0x40,0x90,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_short_d16 v2, off, off offset:16
+// GFX940: scratch_load_short_d16 v2, off, off offset:16 ; encoding: [0x10,0x40,0x90,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_short_d16_hi a2, v4, s6
+// GFX940: scratch_load_short_d16_hi a2, v4, s6    ; encoding: [0x00,0x60,0x94,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_short_d16_hi a2, v4, s6 offset:16
+// GFX940: scratch_load_short_d16_hi a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x94,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_short_d16_hi a2, v4, off
+// GFX940: scratch_load_short_d16_hi a2, v4, off   ; encoding: [0x00,0x60,0x94,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_short_d16_hi a2, v4, off offset:16
+// GFX940: scratch_load_short_d16_hi a2, v4, off offset:16 ; encoding: [0x10,0x60,0x94,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_short_d16_hi a2, off, s6
+// GFX940: scratch_load_short_d16_hi a2, off, s6   ; encoding: [0x00,0x40,0x94,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_short_d16_hi a2, off, s6 offset:16
+// GFX940: scratch_load_short_d16_hi a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x94,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_short_d16_hi a2, off, off
+// GFX940: scratch_load_short_d16_hi a2, off, off  ; encoding: [0x00,0x40,0x94,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_short_d16_hi a2, off, off offset:16
+// GFX940: scratch_load_short_d16_hi a2, off, off offset:16 ; encoding: [0x10,0x40,0x94,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_short_d16_hi v2, v4, s6
+// GFX940: scratch_load_short_d16_hi v2, v4, s6    ; encoding: [0x00,0x60,0x94,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_short_d16_hi v2, v4, s6 offset:16
+// GFX940: scratch_load_short_d16_hi v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x94,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_short_d16_hi v2, v4, off
+// GFX940: scratch_load_short_d16_hi v2, v4, off   ; encoding: [0x00,0x60,0x94,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_short_d16_hi v2, v4, off offset:16
+// GFX940: scratch_load_short_d16_hi v2, v4, off offset:16 ; encoding: [0x10,0x60,0x94,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_short_d16_hi v2, off, s6
+// GFX940: scratch_load_short_d16_hi v2, off, s6   ; encoding: [0x00,0x40,0x94,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_short_d16_hi v2, off, s6 offset:16
+// GFX940: scratch_load_short_d16_hi v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x94,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_short_d16_hi v2, off, off
+// GFX940: scratch_load_short_d16_hi v2, off, off  ; encoding: [0x00,0x40,0x94,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_short_d16_hi v2, off, off offset:16
+// GFX940: scratch_load_short_d16_hi v2, off, off offset:16 ; encoding: [0x10,0x40,0x94,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_sshort a2, v4, s6
+// GFX940: scratch_load_sshort a2, v4, s6          ; encoding: [0x00,0x60,0x4c,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_sshort a2, v4, s6 offset:16
+// GFX940: scratch_load_sshort a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x4c,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_sshort a2, v4, off
+// GFX940: scratch_load_sshort a2, v4, off         ; encoding: [0x00,0x60,0x4c,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_sshort a2, v4, off offset:16
+// GFX940: scratch_load_sshort a2, v4, off offset:16 ; encoding: [0x10,0x60,0x4c,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_sshort a2, off, s6
+// GFX940: scratch_load_sshort a2, off, s6         ; encoding: [0x00,0x40,0x4c,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_sshort a2, off, s6 offset:16
+// GFX940: scratch_load_sshort a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x4c,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_sshort a2, off, off
+// GFX940: scratch_load_sshort a2, off, off        ; encoding: [0x00,0x40,0x4c,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_sshort a2, off, off offset:16
+// GFX940: scratch_load_sshort a2, off, off offset:16 ; encoding: [0x10,0x40,0x4c,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_sshort v2, v4, s6
+// GFX940: scratch_load_sshort v2, v4, s6          ; encoding: [0x00,0x60,0x4c,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_sshort v2, v4, s6 offset:16
+// GFX940: scratch_load_sshort v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x4c,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_sshort v2, v4, off
+// GFX940: scratch_load_sshort v2, v4, off         ; encoding: [0x00,0x60,0x4c,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_sshort v2, v4, off offset:16
+// GFX940: scratch_load_sshort v2, v4, off offset:16 ; encoding: [0x10,0x60,0x4c,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_sshort v2, off, s6
+// GFX940: scratch_load_sshort v2, off, s6         ; encoding: [0x00,0x40,0x4c,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_sshort v2, off, s6 offset:16
+// GFX940: scratch_load_sshort v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x4c,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_sshort v2, off, off
+// GFX940: scratch_load_sshort v2, off, off        ; encoding: [0x00,0x40,0x4c,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_sshort v2, off, off offset:16
+// GFX940: scratch_load_sshort v2, off, off offset:16 ; encoding: [0x10,0x40,0x4c,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_ubyte a2, v4, s6
+// GFX940: scratch_load_ubyte a2, v4, s6           ; encoding: [0x00,0x60,0x40,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_ubyte a2, v4, s6 offset:16
+// GFX940: scratch_load_ubyte a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x40,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_ubyte a2, v4, off
+// GFX940: scratch_load_ubyte a2, v4, off          ; encoding: [0x00,0x60,0x40,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_ubyte a2, v4, off offset:16
+// GFX940: scratch_load_ubyte a2, v4, off offset:16 ; encoding: [0x10,0x60,0x40,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_ubyte a2, off, s6
+// GFX940: scratch_load_ubyte a2, off, s6          ; encoding: [0x00,0x40,0x40,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_ubyte a2, off, s6 offset:16
+// GFX940: scratch_load_ubyte a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x40,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_ubyte a2, off, off
+// GFX940: scratch_load_ubyte a2, off, off         ; encoding: [0x00,0x40,0x40,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_ubyte a2, off, off offset:16
+// GFX940: scratch_load_ubyte a2, off, off offset:16 ; encoding: [0x10,0x40,0x40,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_ubyte v2, v4, s6
+// GFX940: scratch_load_ubyte v2, v4, s6           ; encoding: [0x00,0x60,0x40,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_ubyte v2, v4, s6 offset:16
+// GFX940: scratch_load_ubyte v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x40,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_ubyte v2, v4, off
+// GFX940: scratch_load_ubyte v2, v4, off          ; encoding: [0x00,0x60,0x40,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_ubyte v2, v4, off offset:16
+// GFX940: scratch_load_ubyte v2, v4, off offset:16 ; encoding: [0x10,0x60,0x40,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_ubyte v2, off, s6
+// GFX940: scratch_load_ubyte v2, off, s6          ; encoding: [0x00,0x40,0x40,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_ubyte v2, off, s6 offset:16
+// GFX940: scratch_load_ubyte v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x40,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_ubyte v2, off, off
+// GFX940: scratch_load_ubyte v2, off, off         ; encoding: [0x00,0x40,0x40,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_ubyte v2, off, off offset:16
+// GFX940: scratch_load_ubyte v2, off, off offset:16 ; encoding: [0x10,0x40,0x40,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_ubyte_d16 a2, v4, s6
+// GFX940: scratch_load_ubyte_d16 a2, v4, s6       ; encoding: [0x00,0x60,0x80,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_ubyte_d16 a2, v4, s6 offset:16
+// GFX940: scratch_load_ubyte_d16 a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x80,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_ubyte_d16 a2, v4, off
+// GFX940: scratch_load_ubyte_d16 a2, v4, off      ; encoding: [0x00,0x60,0x80,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_ubyte_d16 a2, v4, off offset:16
+// GFX940: scratch_load_ubyte_d16 a2, v4, off offset:16 ; encoding: [0x10,0x60,0x80,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_ubyte_d16 a2, off, s6
+// GFX940: scratch_load_ubyte_d16 a2, off, s6      ; encoding: [0x00,0x40,0x80,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_ubyte_d16 a2, off, s6 offset:16
+// GFX940: scratch_load_ubyte_d16 a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x80,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_ubyte_d16 a2, off, off
+// GFX940: scratch_load_ubyte_d16 a2, off, off     ; encoding: [0x00,0x40,0x80,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_ubyte_d16 a2, off, off offset:16
+// GFX940: scratch_load_ubyte_d16 a2, off, off offset:16 ; encoding: [0x10,0x40,0x80,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_ubyte_d16 v2, v4, s6
+// GFX940: scratch_load_ubyte_d16 v2, v4, s6       ; encoding: [0x00,0x60,0x80,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_ubyte_d16 v2, v4, s6 offset:16
+// GFX940: scratch_load_ubyte_d16 v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x80,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_ubyte_d16 v2, v4, off
+// GFX940: scratch_load_ubyte_d16 v2, v4, off      ; encoding: [0x00,0x60,0x80,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_ubyte_d16 v2, v4, off offset:16
+// GFX940: scratch_load_ubyte_d16 v2, v4, off offset:16 ; encoding: [0x10,0x60,0x80,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_ubyte_d16 v2, off, s6
+// GFX940: scratch_load_ubyte_d16 v2, off, s6      ; encoding: [0x00,0x40,0x80,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_ubyte_d16 v2, off, s6 offset:16
+// GFX940: scratch_load_ubyte_d16 v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x80,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_ubyte_d16 v2, off, off
+// GFX940: scratch_load_ubyte_d16 v2, off, off     ; encoding: [0x00,0x40,0x80,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_ubyte_d16 v2, off, off offset:16
+// GFX940: scratch_load_ubyte_d16 v2, off, off offset:16 ; encoding: [0x10,0x40,0x80,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_ubyte_d16_hi a2, v4, s6
+// GFX940: scratch_load_ubyte_d16_hi a2, v4, s6    ; encoding: [0x00,0x60,0x84,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_ubyte_d16_hi a2, v4, s6 offset:16
+// GFX940: scratch_load_ubyte_d16_hi a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x84,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_ubyte_d16_hi a2, v4, off
+// GFX940: scratch_load_ubyte_d16_hi a2, v4, off   ; encoding: [0x00,0x60,0x84,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_ubyte_d16_hi a2, v4, off offset:16
+// GFX940: scratch_load_ubyte_d16_hi a2, v4, off offset:16 ; encoding: [0x10,0x60,0x84,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_ubyte_d16_hi a2, off, s6
+// GFX940: scratch_load_ubyte_d16_hi a2, off, s6   ; encoding: [0x00,0x40,0x84,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_ubyte_d16_hi a2, off, s6 offset:16
+// GFX940: scratch_load_ubyte_d16_hi a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x84,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_ubyte_d16_hi a2, off, off
+// GFX940: scratch_load_ubyte_d16_hi a2, off, off  ; encoding: [0x00,0x40,0x84,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_ubyte_d16_hi a2, off, off offset:16
+// GFX940: scratch_load_ubyte_d16_hi a2, off, off offset:16 ; encoding: [0x10,0x40,0x84,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_ubyte_d16_hi v2, v4, s6
+// GFX940: scratch_load_ubyte_d16_hi v2, v4, s6    ; encoding: [0x00,0x60,0x84,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_ubyte_d16_hi v2, v4, s6 offset:16
+// GFX940: scratch_load_ubyte_d16_hi v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x84,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_ubyte_d16_hi v2, v4, off
+// GFX940: scratch_load_ubyte_d16_hi v2, v4, off   ; encoding: [0x00,0x60,0x84,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_ubyte_d16_hi v2, v4, off offset:16
+// GFX940: scratch_load_ubyte_d16_hi v2, v4, off offset:16 ; encoding: [0x10,0x60,0x84,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_ubyte_d16_hi v2, off, s6
+// GFX940: scratch_load_ubyte_d16_hi v2, off, s6   ; encoding: [0x00,0x40,0x84,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_ubyte_d16_hi v2, off, s6 offset:16
+// GFX940: scratch_load_ubyte_d16_hi v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x84,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_ubyte_d16_hi v2, off, off
+// GFX940: scratch_load_ubyte_d16_hi v2, off, off  ; encoding: [0x00,0x40,0x84,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_ubyte_d16_hi v2, off, off offset:16
+// GFX940: scratch_load_ubyte_d16_hi v2, off, off offset:16 ; encoding: [0x10,0x40,0x84,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_ushort a2, v4, s6
+// GFX940: scratch_load_ushort a2, v4, s6          ; encoding: [0x00,0x60,0x48,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_ushort a2, v4, s6 offset:16
+// GFX940: scratch_load_ushort a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x48,0xdc,0x04,0x00,0x86,0x02]
+
+scratch_load_ushort a2, v4, off
+// GFX940: scratch_load_ushort a2, v4, off         ; encoding: [0x00,0x60,0x48,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_ushort a2, v4, off offset:16
+// GFX940: scratch_load_ushort a2, v4, off offset:16 ; encoding: [0x10,0x60,0x48,0xdc,0x04,0x00,0xff,0x02]
+
+scratch_load_ushort a2, off, s6
+// GFX940: scratch_load_ushort a2, off, s6         ; encoding: [0x00,0x40,0x48,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_ushort a2, off, s6 offset:16
+// GFX940: scratch_load_ushort a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x48,0xdc,0x00,0x00,0x86,0x02]
+
+scratch_load_ushort a2, off, off
+// GFX940: scratch_load_ushort a2, off, off        ; encoding: [0x00,0x40,0x48,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_ushort a2, off, off offset:16
+// GFX940: scratch_load_ushort a2, off, off offset:16 ; encoding: [0x10,0x40,0x48,0xdc,0x00,0x00,0xff,0x02]
+
+scratch_load_ushort v2, v4, s6
+// GFX940: scratch_load_ushort v2, v4, s6          ; encoding: [0x00,0x60,0x48,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_ushort v2, v4, s6 offset:16
+// GFX940: scratch_load_ushort v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x48,0xdc,0x04,0x00,0x06,0x02]
+
+scratch_load_ushort v2, v4, off
+// GFX940: scratch_load_ushort v2, v4, off         ; encoding: [0x00,0x60,0x48,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_ushort v2, v4, off offset:16
+// GFX940: scratch_load_ushort v2, v4, off offset:16 ; encoding: [0x10,0x60,0x48,0xdc,0x04,0x00,0x7f,0x02]
+
+scratch_load_ushort v2, off, s6
+// GFX940: scratch_load_ushort v2, off, s6         ; encoding: [0x00,0x40,0x48,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_ushort v2, off, s6 offset:16
+// GFX940: scratch_load_ushort v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x48,0xdc,0x00,0x00,0x06,0x02]
+
+scratch_load_ushort v2, off, off
+// GFX940: scratch_load_ushort v2, off, off        ; encoding: [0x00,0x40,0x48,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_load_ushort v2, off, off offset:16
+// GFX940: scratch_load_ushort v2, off, off offset:16 ; encoding: [0x10,0x40,0x48,0xdc,0x00,0x00,0x7f,0x02]
+
+scratch_store_byte v4, a2, s6
+// GFX940: scratch_store_byte v4, a2, s6           ; encoding: [0x00,0x60,0x60,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_byte v4, a2, s6 offset:16
+// GFX940: scratch_store_byte v4, a2, s6 offset:16 ; encoding: [0x10,0x60,0x60,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_byte v4, a2, off
+// GFX940: scratch_store_byte v4, a2, off          ; encoding: [0x00,0x60,0x60,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_byte v4, a2, off offset:16
+// GFX940: scratch_store_byte v4, a2, off offset:16 ; encoding: [0x10,0x60,0x60,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_byte off, a2, s6
+// GFX940: scratch_store_byte off, a2, s6          ; encoding: [0x00,0x40,0x60,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_byte off, a2, s6 offset:16
+// GFX940: scratch_store_byte off, a2, s6 offset:16 ; encoding: [0x10,0x40,0x60,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_byte off, a2, off
+// GFX940: scratch_store_byte off, a2, off         ; encoding: [0x00,0x40,0x60,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_byte off, a2, off offset:16
+// GFX940: scratch_store_byte off, a2, off offset:16 ; encoding: [0x10,0x40,0x60,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_byte v4, v2, s6
+// GFX940: scratch_store_byte v4, v2, s6           ; encoding: [0x00,0x60,0x60,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_byte v4, v2, s6 offset:16
+// GFX940: scratch_store_byte v4, v2, s6 offset:16 ; encoding: [0x10,0x60,0x60,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_byte v4, v2, off
+// GFX940: scratch_store_byte v4, v2, off          ; encoding: [0x00,0x60,0x60,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_byte v4, v2, off offset:16
+// GFX940: scratch_store_byte v4, v2, off offset:16 ; encoding: [0x10,0x60,0x60,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_byte off, v2, s6
+// GFX940: scratch_store_byte off, v2, s6          ; encoding: [0x00,0x40,0x60,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_byte off, v2, s6 offset:16
+// GFX940: scratch_store_byte off, v2, s6 offset:16 ; encoding: [0x10,0x40,0x60,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_byte off, v2, off
+// GFX940: scratch_store_byte off, v2, off         ; encoding: [0x00,0x40,0x60,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_byte off, v2, off offset:16
+// GFX940: scratch_store_byte off, v2, off offset:16 ; encoding: [0x10,0x40,0x60,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_byte_d16_hi v4, a2, s6
+// GFX940: scratch_store_byte_d16_hi v4, a2, s6    ; encoding: [0x00,0x60,0x64,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_byte_d16_hi v4, a2, s6 offset:16
+// GFX940: scratch_store_byte_d16_hi v4, a2, s6 offset:16 ; encoding: [0x10,0x60,0x64,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_byte_d16_hi v4, a2, off
+// GFX940: scratch_store_byte_d16_hi v4, a2, off   ; encoding: [0x00,0x60,0x64,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_byte_d16_hi v4, a2, off offset:16
+// GFX940: scratch_store_byte_d16_hi v4, a2, off offset:16 ; encoding: [0x10,0x60,0x64,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_byte_d16_hi off, a2, s6
+// GFX940: scratch_store_byte_d16_hi off, a2, s6   ; encoding: [0x00,0x40,0x64,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_byte_d16_hi off, a2, s6 offset:16
+// GFX940: scratch_store_byte_d16_hi off, a2, s6 offset:16 ; encoding: [0x10,0x40,0x64,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_byte_d16_hi off, a2, off
+// GFX940: scratch_store_byte_d16_hi off, a2, off  ; encoding: [0x00,0x40,0x64,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_byte_d16_hi off, a2, off offset:16
+// GFX940: scratch_store_byte_d16_hi off, a2, off offset:16 ; encoding: [0x10,0x40,0x64,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_byte_d16_hi v4, v2, s6
+// GFX940: scratch_store_byte_d16_hi v4, v2, s6    ; encoding: [0x00,0x60,0x64,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_byte_d16_hi v4, v2, s6 offset:16
+// GFX940: scratch_store_byte_d16_hi v4, v2, s6 offset:16 ; encoding: [0x10,0x60,0x64,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_byte_d16_hi v4, v2, off
+// GFX940: scratch_store_byte_d16_hi v4, v2, off   ; encoding: [0x00,0x60,0x64,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_byte_d16_hi v4, v2, off offset:16
+// GFX940: scratch_store_byte_d16_hi v4, v2, off offset:16 ; encoding: [0x10,0x60,0x64,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_byte_d16_hi off, v2, s6
+// GFX940: scratch_store_byte_d16_hi off, v2, s6   ; encoding: [0x00,0x40,0x64,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_byte_d16_hi off, v2, s6 offset:16
+// GFX940: scratch_store_byte_d16_hi off, v2, s6 offset:16 ; encoding: [0x10,0x40,0x64,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_byte_d16_hi off, v2, off
+// GFX940: scratch_store_byte_d16_hi off, v2, off  ; encoding: [0x00,0x40,0x64,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_byte_d16_hi off, v2, off offset:16
+// GFX940: scratch_store_byte_d16_hi off, v2, off offset:16 ; encoding: [0x10,0x40,0x64,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_dword v4, a2, s6
+// GFX940: scratch_store_dword v4, a2, s6          ; encoding: [0x00,0x60,0x70,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_dword v4, a2, s6 offset:16
+// GFX940: scratch_store_dword v4, a2, s6 offset:16 ; encoding: [0x10,0x60,0x70,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_dword v4, a2, off
+// GFX940: scratch_store_dword v4, a2, off         ; encoding: [0x00,0x60,0x70,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_dword v4, a2, off offset:16
+// GFX940: scratch_store_dword v4, a2, off offset:16 ; encoding: [0x10,0x60,0x70,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_dword off, a2, s6
+// GFX940: scratch_store_dword off, a2, s6         ; encoding: [0x00,0x40,0x70,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_dword off, a2, s6 offset:16
+// GFX940: scratch_store_dword off, a2, s6 offset:16 ; encoding: [0x10,0x40,0x70,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_dword off, a2, off
+// GFX940: scratch_store_dword off, a2, off        ; encoding: [0x00,0x40,0x70,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_dword off, a2, off offset:16
+// GFX940: scratch_store_dword off, a2, off offset:16 ; encoding: [0x10,0x40,0x70,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_dword v4, v2, s6
+// GFX940: scratch_store_dword v4, v2, s6          ; encoding: [0x00,0x60,0x70,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_dword v4, v2, s6 offset:16
+// GFX940: scratch_store_dword v4, v2, s6 offset:16 ; encoding: [0x10,0x60,0x70,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_dword v4, v2, off
+// GFX940: scratch_store_dword v4, v2, off         ; encoding: [0x00,0x60,0x70,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_dword v4, v2, off offset:16
+// GFX940: scratch_store_dword v4, v2, off offset:16 ; encoding: [0x10,0x60,0x70,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_dword off, v2, s6
+// GFX940: scratch_store_dword off, v2, s6         ; encoding: [0x00,0x40,0x70,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_dword off, v2, s6 offset:16
+// GFX940: scratch_store_dword off, v2, s6 offset:16 ; encoding: [0x10,0x40,0x70,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_dword off, v2, off
+// GFX940: scratch_store_dword off, v2, off        ; encoding: [0x00,0x40,0x70,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_dword off, v2, off offset:16
+// GFX940: scratch_store_dword off, v2, off offset:16 ; encoding: [0x10,0x40,0x70,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_dwordx2 v4, a[2:3], s6
+// GFX940: scratch_store_dwordx2 v4, a[2:3], s6    ; encoding: [0x00,0x60,0x74,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_dwordx2 v4, a[2:3], s6 offset:16
+// GFX940: scratch_store_dwordx2 v4, a[2:3], s6 offset:16 ; encoding: [0x10,0x60,0x74,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_dwordx2 v4, a[2:3], off
+// GFX940: scratch_store_dwordx2 v4, a[2:3], off   ; encoding: [0x00,0x60,0x74,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_dwordx2 v4, a[2:3], off offset:16
+// GFX940: scratch_store_dwordx2 v4, a[2:3], off offset:16 ; encoding: [0x10,0x60,0x74,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_dwordx2 off, a[2:3], s6
+// GFX940: scratch_store_dwordx2 off, a[2:3], s6   ; encoding: [0x00,0x40,0x74,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_dwordx2 off, a[2:3], s6 offset:16
+// GFX940: scratch_store_dwordx2 off, a[2:3], s6 offset:16 ; encoding: [0x10,0x40,0x74,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_dwordx2 off, a[2:3], off
+// GFX940: scratch_store_dwordx2 off, a[2:3], off  ; encoding: [0x00,0x40,0x74,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_dwordx2 off, a[2:3], off offset:16
+// GFX940: scratch_store_dwordx2 off, a[2:3], off offset:16 ; encoding: [0x10,0x40,0x74,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_dwordx2 v4, v[2:3], s6
+// GFX940: scratch_store_dwordx2 v4, v[2:3], s6    ; encoding: [0x00,0x60,0x74,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_dwordx2 v4, v[2:3], s6 offset:16
+// GFX940: scratch_store_dwordx2 v4, v[2:3], s6 offset:16 ; encoding: [0x10,0x60,0x74,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_dwordx2 v4, v[2:3], off
+// GFX940: scratch_store_dwordx2 v4, v[2:3], off   ; encoding: [0x00,0x60,0x74,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_dwordx2 v4, v[2:3], off offset:16
+// GFX940: scratch_store_dwordx2 v4, v[2:3], off offset:16 ; encoding: [0x10,0x60,0x74,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_dwordx2 off, v[2:3], s6
+// GFX940: scratch_store_dwordx2 off, v[2:3], s6   ; encoding: [0x00,0x40,0x74,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_dwordx2 off, v[2:3], s6 offset:16
+// GFX940: scratch_store_dwordx2 off, v[2:3], s6 offset:16 ; encoding: [0x10,0x40,0x74,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_dwordx2 off, v[2:3], off
+// GFX940: scratch_store_dwordx2 off, v[2:3], off  ; encoding: [0x00,0x40,0x74,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_dwordx2 off, v[2:3], off offset:16
+// GFX940: scratch_store_dwordx2 off, v[2:3], off offset:16 ; encoding: [0x10,0x40,0x74,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_dwordx3 v4, a[2:4], s6
+// GFX940: scratch_store_dwordx3 v4, a[2:4], s6    ; encoding: [0x00,0x60,0x78,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_dwordx3 v4, a[2:4], s6 offset:16
+// GFX940: scratch_store_dwordx3 v4, a[2:4], s6 offset:16 ; encoding: [0x10,0x60,0x78,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_dwordx3 v4, a[2:4], off
+// GFX940: scratch_store_dwordx3 v4, a[2:4], off   ; encoding: [0x00,0x60,0x78,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_dwordx3 v4, a[2:4], off offset:16
+// GFX940: scratch_store_dwordx3 v4, a[2:4], off offset:16 ; encoding: [0x10,0x60,0x78,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_dwordx3 off, a[2:4], s6
+// GFX940: scratch_store_dwordx3 off, a[2:4], s6   ; encoding: [0x00,0x40,0x78,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_dwordx3 off, a[2:4], s6 offset:16
+// GFX940: scratch_store_dwordx3 off, a[2:4], s6 offset:16 ; encoding: [0x10,0x40,0x78,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_dwordx3 off, a[2:4], off
+// GFX940: scratch_store_dwordx3 off, a[2:4], off  ; encoding: [0x00,0x40,0x78,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_dwordx3 off, a[2:4], off offset:16
+// GFX940: scratch_store_dwordx3 off, a[2:4], off offset:16 ; encoding: [0x10,0x40,0x78,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_dwordx3 v4, v[2:4], s6
+// GFX940: scratch_store_dwordx3 v4, v[2:4], s6    ; encoding: [0x00,0x60,0x78,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_dwordx3 v4, v[2:4], s6 offset:16
+// GFX940: scratch_store_dwordx3 v4, v[2:4], s6 offset:16 ; encoding: [0x10,0x60,0x78,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_dwordx3 v4, v[2:4], off
+// GFX940: scratch_store_dwordx3 v4, v[2:4], off   ; encoding: [0x00,0x60,0x78,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_dwordx3 v4, v[2:4], off offset:16
+// GFX940: scratch_store_dwordx3 v4, v[2:4], off offset:16 ; encoding: [0x10,0x60,0x78,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_dwordx3 off, v[2:4], s6
+// GFX940: scratch_store_dwordx3 off, v[2:4], s6   ; encoding: [0x00,0x40,0x78,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_dwordx3 off, v[2:4], s6 offset:16
+// GFX940: scratch_store_dwordx3 off, v[2:4], s6 offset:16 ; encoding: [0x10,0x40,0x78,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_dwordx3 off, v[2:4], off
+// GFX940: scratch_store_dwordx3 off, v[2:4], off  ; encoding: [0x00,0x40,0x78,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_dwordx3 off, v[2:4], off offset:16
+// GFX940: scratch_store_dwordx3 off, v[2:4], off offset:16 ; encoding: [0x10,0x40,0x78,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_dwordx4 v4, a[2:5], s6
+// GFX940: scratch_store_dwordx4 v4, a[2:5], s6    ; encoding: [0x00,0x60,0x7c,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_dwordx4 v4, a[2:5], s6 offset:16
+// GFX940: scratch_store_dwordx4 v4, a[2:5], s6 offset:16 ; encoding: [0x10,0x60,0x7c,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_dwordx4 v4, a[2:5], off
+// GFX940: scratch_store_dwordx4 v4, a[2:5], off   ; encoding: [0x00,0x60,0x7c,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_dwordx4 v4, a[2:5], off offset:16
+// GFX940: scratch_store_dwordx4 v4, a[2:5], off offset:16 ; encoding: [0x10,0x60,0x7c,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_dwordx4 off, a[2:5], s6
+// GFX940: scratch_store_dwordx4 off, a[2:5], s6   ; encoding: [0x00,0x40,0x7c,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_dwordx4 off, a[2:5], s6 offset:16
+// GFX940: scratch_store_dwordx4 off, a[2:5], s6 offset:16 ; encoding: [0x10,0x40,0x7c,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_dwordx4 off, a[2:5], off
+// GFX940: scratch_store_dwordx4 off, a[2:5], off  ; encoding: [0x00,0x40,0x7c,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_dwordx4 off, a[2:5], off offset:16
+// GFX940: scratch_store_dwordx4 off, a[2:5], off offset:16 ; encoding: [0x10,0x40,0x7c,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_dwordx4 v4, v[2:5], s6
+// GFX940: scratch_store_dwordx4 v4, v[2:5], s6    ; encoding: [0x00,0x60,0x7c,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_dwordx4 v4, v[2:5], s6 offset:16
+// GFX940: scratch_store_dwordx4 v4, v[2:5], s6 offset:16 ; encoding: [0x10,0x60,0x7c,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_dwordx4 v4, v[2:5], off
+// GFX940: scratch_store_dwordx4 v4, v[2:5], off   ; encoding: [0x00,0x60,0x7c,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_dwordx4 v4, v[2:5], off offset:16
+// GFX940: scratch_store_dwordx4 v4, v[2:5], off offset:16 ; encoding: [0x10,0x60,0x7c,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_dwordx4 off, v[2:5], s6
+// GFX940: scratch_store_dwordx4 off, v[2:5], s6   ; encoding: [0x00,0x40,0x7c,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_dwordx4 off, v[2:5], s6 offset:16
+// GFX940: scratch_store_dwordx4 off, v[2:5], s6 offset:16 ; encoding: [0x10,0x40,0x7c,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_dwordx4 off, v[2:5], off
+// GFX940: scratch_store_dwordx4 off, v[2:5], off  ; encoding: [0x00,0x40,0x7c,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_dwordx4 off, v[2:5], off offset:16
+// GFX940: scratch_store_dwordx4 off, v[2:5], off offset:16 ; encoding: [0x10,0x40,0x7c,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_short v4, a2, s6
+// GFX940: scratch_store_short v4, a2, s6          ; encoding: [0x00,0x60,0x68,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_short v4, a2, s6 offset:16
+// GFX940: scratch_store_short v4, a2, s6 offset:16 ; encoding: [0x10,0x60,0x68,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_short v4, a2, off
+// GFX940: scratch_store_short v4, a2, off         ; encoding: [0x00,0x60,0x68,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_short v4, a2, off offset:16
+// GFX940: scratch_store_short v4, a2, off offset:16 ; encoding: [0x10,0x60,0x68,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_short off, a2, s6
+// GFX940: scratch_store_short off, a2, s6         ; encoding: [0x00,0x40,0x68,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_short off, a2, s6 offset:16
+// GFX940: scratch_store_short off, a2, s6 offset:16 ; encoding: [0x10,0x40,0x68,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_short off, a2, off
+// GFX940: scratch_store_short off, a2, off        ; encoding: [0x00,0x40,0x68,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_short off, a2, off offset:16
+// GFX940: scratch_store_short off, a2, off offset:16 ; encoding: [0x10,0x40,0x68,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_short v4, v2, s6
+// GFX940: scratch_store_short v4, v2, s6          ; encoding: [0x00,0x60,0x68,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_short v4, v2, s6 offset:16
+// GFX940: scratch_store_short v4, v2, s6 offset:16 ; encoding: [0x10,0x60,0x68,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_short v4, v2, off
+// GFX940: scratch_store_short v4, v2, off         ; encoding: [0x00,0x60,0x68,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_short v4, v2, off offset:16
+// GFX940: scratch_store_short v4, v2, off offset:16 ; encoding: [0x10,0x60,0x68,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_short off, v2, s6
+// GFX940: scratch_store_short off, v2, s6         ; encoding: [0x00,0x40,0x68,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_short off, v2, s6 offset:16
+// GFX940: scratch_store_short off, v2, s6 offset:16 ; encoding: [0x10,0x40,0x68,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_short off, v2, off
+// GFX940: scratch_store_short off, v2, off        ; encoding: [0x00,0x40,0x68,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_short off, v2, off offset:16
+// GFX940: scratch_store_short off, v2, off offset:16 ; encoding: [0x10,0x40,0x68,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_short_d16_hi v4, a2, s6
+// GFX940: scratch_store_short_d16_hi v4, a2, s6   ; encoding: [0x00,0x60,0x6c,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_short_d16_hi v4, a2, s6 offset:16
+// GFX940: scratch_store_short_d16_hi v4, a2, s6 offset:16 ; encoding: [0x10,0x60,0x6c,0xdc,0x04,0x02,0x86,0x00]
+
+scratch_store_short_d16_hi v4, a2, off
+// GFX940: scratch_store_short_d16_hi v4, a2, off  ; encoding: [0x00,0x60,0x6c,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_short_d16_hi v4, a2, off offset:16
+// GFX940: scratch_store_short_d16_hi v4, a2, off offset:16 ; encoding: [0x10,0x60,0x6c,0xdc,0x04,0x02,0xff,0x00]
+
+scratch_store_short_d16_hi off, a2, s6
+// GFX940: scratch_store_short_d16_hi off, a2, s6  ; encoding: [0x00,0x40,0x6c,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_short_d16_hi off, a2, s6 offset:16
+// GFX940: scratch_store_short_d16_hi off, a2, s6 offset:16 ; encoding: [0x10,0x40,0x6c,0xdc,0x00,0x02,0x86,0x00]
+
+scratch_store_short_d16_hi off, a2, off
+// GFX940: scratch_store_short_d16_hi off, a2, off ; encoding: [0x00,0x40,0x6c,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_short_d16_hi off, a2, off offset:16
+// GFX940: scratch_store_short_d16_hi off, a2, off offset:16 ; encoding: [0x10,0x40,0x6c,0xdc,0x00,0x02,0xff,0x00]
+
+scratch_store_short_d16_hi v4, v2, s6
+// GFX940: scratch_store_short_d16_hi v4, v2, s6   ; encoding: [0x00,0x60,0x6c,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_short_d16_hi v4, v2, s6 offset:16
+// GFX940: scratch_store_short_d16_hi v4, v2, s6 offset:16 ; encoding: [0x10,0x60,0x6c,0xdc,0x04,0x02,0x06,0x00]
+
+scratch_store_short_d16_hi v4, v2, off
+// GFX940: scratch_store_short_d16_hi v4, v2, off  ; encoding: [0x00,0x60,0x6c,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_short_d16_hi v4, v2, off offset:16
+// GFX940: scratch_store_short_d16_hi v4, v2, off offset:16 ; encoding: [0x10,0x60,0x6c,0xdc,0x04,0x02,0x7f,0x00]
+
+scratch_store_short_d16_hi off, v2, s6
+// GFX940: scratch_store_short_d16_hi off, v2, s6  ; encoding: [0x00,0x40,0x6c,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_short_d16_hi off, v2, s6 offset:16
+// GFX940: scratch_store_short_d16_hi off, v2, s6 offset:16 ; encoding: [0x10,0x40,0x6c,0xdc,0x00,0x02,0x06,0x00]
+
+scratch_store_short_d16_hi off, v2, off
+// GFX940: scratch_store_short_d16_hi off, v2, off ; encoding: [0x00,0x40,0x6c,0xdc,0x00,0x02,0x7f,0x00]
+
+scratch_store_short_d16_hi off, v2, off offset:16
+// GFX940: scratch_store_short_d16_hi off, v2, off offset:16 ; encoding: [0x10,0x40,0x6c,0xdc,0x00,0x02,0x7f,0x00]

llvm/test/MC/AMDGPU/flat-scratch-instructions.s

 
 scratch_store_dword off, v2, s1 offset:12
 // GFX10: encoding: [0x0c,0x40,0x70,0xdc,0x00,0x02,0x01,0x00]
 // GFX9: scratch_store_dword off, v2, s1 offset:12 ; encoding: [0x0c,0x40,0x70,0xdc,0x00,0x02,0x01,0x00]
 // VI-ERR: error: instruction not supported on this GPU
 
 // FIXME: Should error about multiple offsets
 scratch_load_dword v1, v2, s1
-// GFX10-ERR: error: invalid operand for instruction
-// GFX9-ERR: error: invalid operand for instruction
+// GFX10-ERR: error: operands are not valid for this GPU or mode
+// GFX9-ERR: error: operands are not valid for this GPU or mode
 // VI-ERR: error: instruction not supported on this GPU
 
 scratch_load_dword v1, v2, s1 offset:32
-// GFX10-ERR: error: invalid operand for instruction
-// GFX9-ERR: error: invalid operand for instruction
+// GFX10-ERR: error: operands are not valid for this GPU or mode
+// GFX9-ERR: error: operands are not valid for this GPU or mode
 // VI-ERR: error: instruction not supported on this GPU
 
 scratch_store_dword v1, v2, s1
-// GFX10-ERR: error: invalid operand for instruction
-// GFX9-ERR: error: invalid operand for instruction
+// GFX10-ERR: error: operands are not valid for this GPU or mode
+// GFX9-ERR: error: operands are not valid for this GPU or mode
 // VI-ERR: error: instruction not supported on this GPU
 
 scratch_store_dword v1, v2, s1 offset:32
-// GFX10-ERR: error: invalid operand for instruction
-// GFX9-ERR: error: invalid operand for instruction
+// GFX10-ERR: error: operands are not valid for this GPU or mode
+// GFX9-ERR: error: operands are not valid for this GPU or mode
 // VI-ERR: error: instruction not supported on this GPU
 
 scratch_load_dword v1, off, exec_hi
 // GFX10-ERR: error: invalid operand for instruction
 // GFX9-ERR: error: invalid operand for instruction
 // VI-ERR: error: instruction not supported on this GPU
 
 scratch_store_dword off, v2, exec_hi

llvm/test/MC/Disassembler/AMDGPU/flat-scratch-gfx940.txt

+# RUN: llvm-mc -arch=amdgcn -mcpu=gfx940 -show-encoding -disassemble %s | FileCheck -check-prefix=GFX940 %s
+
+# GFX940: scratch_load_dword a2, v4, s6 ; encoding: [0x00,0x60,0x50,0xdc,0x04,0x00,0x86,0x02]
+0x00,0x60,0x50,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_dword a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x50,0xdc,0x04,0x00,0x86,0x02]
+0x10,0x60,0x50,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_dword a2, v4, off ; encoding: [0x00,0x60,0x50,0xdc,0x04,0x00,0xff,0x02]
+0x00,0x60,0x50,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_dword a2, v4, off offset:16 ; encoding: [0x10,0x60,0x50,0xdc,0x04,0x00,0xff,0x02]
+0x10,0x60,0x50,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_dword a2, off, s6 ; encoding: [0x00,0x40,0x50,0xdc,0x00,0x00,0x86,0x02]
+0x00,0x40,0x50,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_dword a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x50,0xdc,0x00,0x00,0x86,0x02]
+0x10,0x40,0x50,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_dword a2, off, off ; encoding: [0x00,0x40,0x50,0xdc,0x00,0x00,0xff,0x02]
+0x00,0x40,0x50,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_dword a2, off, off offset:16 ; encoding: [0x10,0x40,0x50,0xdc,0x00,0x00,0xff,0x02]
+0x10,0x40,0x50,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_dword v2, v4, s6 ; encoding: [0x00,0x60,0x50,0xdc,0x04,0x00,0x06,0x02]
+0x00,0x60,0x50,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_dword v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x50,0xdc,0x04,0x00,0x06,0x02]
+0x10,0x60,0x50,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_dword v2, v4, off ; encoding: [0x00,0x60,0x50,0xdc,0x04,0x00,0x7f,0x02]
+0x00,0x60,0x50,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dword v2, v4, off offset:16 ; encoding: [0x10,0x60,0x50,0xdc,0x04,0x00,0x7f,0x02]
+0x10,0x60,0x50,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dword v2, off, s6 ; encoding: [0x00,0x40,0x50,0xdc,0x00,0x00,0x06,0x02]
+0x00,0x40,0x50,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_dword v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x50,0xdc,0x00,0x00,0x06,0x02]
+0x10,0x40,0x50,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_dword v2, off, off ; encoding: [0x00,0x40,0x50,0xdc,0x00,0x00,0x7f,0x02]
+0x00,0x40,0x50,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dword v2, off, off offset:16 ; encoding: [0x10,0x40,0x50,0xdc,0x00,0x00,0x7f,0x02]
+0x10,0x40,0x50,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dwordx2 a[2:3], v4, s6 ; encoding: [0x00,0x60,0x54,0xdc,0x04,0x00,0x86,0x02]
+0x00,0x60,0x54,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_dwordx2 a[2:3], v4, s6 offset:16 ; encoding: [0x10,0x60,0x54,0xdc,0x04,0x00,0x86,0x02]
+0x10,0x60,0x54,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_dwordx2 a[2:3], v4, off ; encoding: [0x00,0x60,0x54,0xdc,0x04,0x00,0xff,0x02]
+0x00,0x60,0x54,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_dwordx2 a[2:3], v4, off offset:16 ; encoding: [0x10,0x60,0x54,0xdc,0x04,0x00,0xff,0x02]
+0x10,0x60,0x54,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_dwordx2 a[2:3], off, s6 ; encoding: [0x00,0x40,0x54,0xdc,0x00,0x00,0x86,0x02]
+0x00,0x40,0x54,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_dwordx2 a[2:3], off, s6 offset:16 ; encoding: [0x10,0x40,0x54,0xdc,0x00,0x00,0x86,0x02]
+0x10,0x40,0x54,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_dwordx2 a[2:3], off, off ; encoding: [0x00,0x40,0x54,0xdc,0x00,0x00,0xff,0x02]
+0x00,0x40,0x54,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_dwordx2 a[2:3], off, off offset:16 ; encoding: [0x10,0x40,0x54,0xdc,0x00,0x00,0xff,0x02]
+0x10,0x40,0x54,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_dwordx2 v[2:3], v4, s6 ; encoding: [0x00,0x60,0x54,0xdc,0x04,0x00,0x06,0x02]
+0x00,0x60,0x54,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_dwordx2 v[2:3], v4, s6 offset:16 ; encoding: [0x10,0x60,0x54,0xdc,0x04,0x00,0x06,0x02]
+0x10,0x60,0x54,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_dwordx2 v[2:3], v4, off ; encoding: [0x00,0x60,0x54,0xdc,0x04,0x00,0x7f,0x02]
+0x00,0x60,0x54,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dwordx2 v[2:3], v4, off offset:16 ; encoding: [0x10,0x60,0x54,0xdc,0x04,0x00,0x7f,0x02]
+0x10,0x60,0x54,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dwordx2 v[2:3], off, s6 ; encoding: [0x00,0x40,0x54,0xdc,0x00,0x00,0x06,0x02]
+0x00,0x40,0x54,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_dwordx2 v[2:3], off, s6 offset:16 ; encoding: [0x10,0x40,0x54,0xdc,0x00,0x00,0x06,0x02]
+0x10,0x40,0x54,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_dwordx2 v[2:3], off, off ; encoding: [0x00,0x40,0x54,0xdc,0x00,0x00,0x7f,0x02]
+0x00,0x40,0x54,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dwordx2 v[2:3], off, off offset:16 ; encoding: [0x10,0x40,0x54,0xdc,0x00,0x00,0x7f,0x02]
+0x10,0x40,0x54,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dwordx3 a[2:4], v4, s6 ; encoding: [0x00,0x60,0x58,0xdc,0x04,0x00,0x86,0x02]
+0x00,0x60,0x58,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_dwordx3 a[2:4], v4, s6 offset:16 ; encoding: [0x10,0x60,0x58,0xdc,0x04,0x00,0x86,0x02]
+0x10,0x60,0x58,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_dwordx3 a[2:4], v4, off ; encoding: [0x00,0x60,0x58,0xdc,0x04,0x00,0xff,0x02]
+0x00,0x60,0x58,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_dwordx3 a[2:4], v4, off offset:16 ; encoding: [0x10,0x60,0x58,0xdc,0x04,0x00,0xff,0x02]
+0x10,0x60,0x58,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_dwordx3 a[2:4], off, s6 ; encoding: [0x00,0x40,0x58,0xdc,0x00,0x00,0x86,0x02]
+0x00,0x40,0x58,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_dwordx3 a[2:4], off, s6 offset:16 ; encoding: [0x10,0x40,0x58,0xdc,0x00,0x00,0x86,0x02]
+0x10,0x40,0x58,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_dwordx3 a[2:4], off, off ; encoding: [0x00,0x40,0x58,0xdc,0x00,0x00,0xff,0x02]
+0x00,0x40,0x58,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_dwordx3 a[2:4], off, off offset:16 ; encoding: [0x10,0x40,0x58,0xdc,0x00,0x00,0xff,0x02]
+0x10,0x40,0x58,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_dwordx3 v[2:4], v4, s6 ; encoding: [0x00,0x60,0x58,0xdc,0x04,0x00,0x06,0x02]
+0x00,0x60,0x58,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_dwordx3 v[2:4], v4, s6 offset:16 ; encoding: [0x10,0x60,0x58,0xdc,0x04,0x00,0x06,0x02]
+0x10,0x60,0x58,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_dwordx3 v[2:4], v4, off ; encoding: [0x00,0x60,0x58,0xdc,0x04,0x00,0x7f,0x02]
+0x00,0x60,0x58,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dwordx3 v[2:4], v4, off offset:16 ; encoding: [0x10,0x60,0x58,0xdc,0x04,0x00,0x7f,0x02]
+0x10,0x60,0x58,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dwordx3 v[2:4], off, s6 ; encoding: [0x00,0x40,0x58,0xdc,0x00,0x00,0x06,0x02]
+0x00,0x40,0x58,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_dwordx3 v[2:4], off, s6 offset:16 ; encoding: [0x10,0x40,0x58,0xdc,0x00,0x00,0x06,0x02]
+0x10,0x40,0x58,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_dwordx3 v[2:4], off, off ; encoding: [0x00,0x40,0x58,0xdc,0x00,0x00,0x7f,0x02]
+0x00,0x40,0x58,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dwordx3 v[2:4], off, off offset:16 ; encoding: [0x10,0x40,0x58,0xdc,0x00,0x00,0x7f,0x02]
+0x10,0x40,0x58,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dwordx4 a[2:5], v4, s6 ; encoding: [0x00,0x60,0x5c,0xdc,0x04,0x00,0x86,0x02]
+0x00,0x60,0x5c,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_dwordx4 a[2:5], v4, s6 offset:16 ; encoding: [0x10,0x60,0x5c,0xdc,0x04,0x00,0x86,0x02]
+0x10,0x60,0x5c,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_dwordx4 a[2:5], v4, off ; encoding: [0x00,0x60,0x5c,0xdc,0x04,0x00,0xff,0x02]
+0x00,0x60,0x5c,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_dwordx4 a[2:5], v4, off offset:16 ; encoding: [0x10,0x60,0x5c,0xdc,0x04,0x00,0xff,0x02]
+0x10,0x60,0x5c,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_dwordx4 a[2:5], off, s6 ; encoding: [0x00,0x40,0x5c,0xdc,0x00,0x00,0x86,0x02]
+0x00,0x40,0x5c,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_dwordx4 a[2:5], off, s6 offset:16 ; encoding: [0x10,0x40,0x5c,0xdc,0x00,0x00,0x86,0x02]
+0x10,0x40,0x5c,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_dwordx4 a[2:5], off, off ; encoding: [0x00,0x40,0x5c,0xdc,0x00,0x00,0xff,0x02]
+0x00,0x40,0x5c,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_dwordx4 a[2:5], off, off offset:16 ; encoding: [0x10,0x40,0x5c,0xdc,0x00,0x00,0xff,0x02]
+0x10,0x40,0x5c,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_dwordx4 v[2:5], v4, s6 ; encoding: [0x00,0x60,0x5c,0xdc,0x04,0x00,0x06,0x02]
+0x00,0x60,0x5c,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_dwordx4 v[2:5], v4, s6 offset:16 ; encoding: [0x10,0x60,0x5c,0xdc,0x04,0x00,0x06,0x02]
+0x10,0x60,0x5c,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_dwordx4 v[2:5], v4, off ; encoding: [0x00,0x60,0x5c,0xdc,0x04,0x00,0x7f,0x02]
+0x00,0x60,0x5c,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dwordx4 v[2:5], v4, off offset:16 ; encoding: [0x10,0x60,0x5c,0xdc,0x04,0x00,0x7f,0x02]
+0x10,0x60,0x5c,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dwordx4 v[2:5], off, s6 ; encoding: [0x00,0x40,0x5c,0xdc,0x00,0x00,0x06,0x02]
+0x00,0x40,0x5c,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_dwordx4 v[2:5], off, s6 offset:16 ; encoding: [0x10,0x40,0x5c,0xdc,0x00,0x00,0x06,0x02]
+0x10,0x40,0x5c,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_dwordx4 v[2:5], off, off ; encoding: [0x00,0x40,0x5c,0xdc,0x00,0x00,0x7f,0x02]
+0x00,0x40,0x5c,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_dwordx4 v[2:5], off, off offset:16 ; encoding: [0x10,0x40,0x5c,0xdc,0x00,0x00,0x7f,0x02]
+0x10,0x40,0x5c,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sbyte a2, v4, s6 ; encoding: [0x00,0x60,0x44,0xdc,0x04,0x00,0x86,0x02]
+0x00,0x60,0x44,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_sbyte a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x44,0xdc,0x04,0x00,0x86,0x02]
+0x10,0x60,0x44,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_sbyte a2, v4, off ; encoding: [0x00,0x60,0x44,0xdc,0x04,0x00,0xff,0x02]
+0x00,0x60,0x44,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_sbyte a2, v4, off offset:16 ; encoding: [0x10,0x60,0x44,0xdc,0x04,0x00,0xff,0x02]
+0x10,0x60,0x44,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_sbyte a2, off, s6 ; encoding: [0x00,0x40,0x44,0xdc,0x00,0x00,0x86,0x02]
+0x00,0x40,0x44,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_sbyte a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x44,0xdc,0x00,0x00,0x86,0x02]
+0x10,0x40,0x44,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_sbyte a2, off, off ; encoding: [0x00,0x40,0x44,0xdc,0x00,0x00,0xff,0x02]
+0x00,0x40,0x44,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_sbyte a2, off, off offset:16 ; encoding: [0x10,0x40,0x44,0xdc,0x00,0x00,0xff,0x02]
+0x10,0x40,0x44,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_sbyte v2, v4, s6 ; encoding: [0x00,0x60,0x44,0xdc,0x04,0x00,0x06,0x02]
+0x00,0x60,0x44,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_sbyte v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x44,0xdc,0x04,0x00,0x06,0x02]
+0x10,0x60,0x44,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_sbyte v2, v4, off ; encoding: [0x00,0x60,0x44,0xdc,0x04,0x00,0x7f,0x02]
+0x00,0x60,0x44,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sbyte v2, v4, off offset:16 ; encoding: [0x10,0x60,0x44,0xdc,0x04,0x00,0x7f,0x02]
+0x10,0x60,0x44,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sbyte v2, off, s6 ; encoding: [0x00,0x40,0x44,0xdc,0x00,0x00,0x06,0x02]
+0x00,0x40,0x44,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_sbyte v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x44,0xdc,0x00,0x00,0x06,0x02]
+0x10,0x40,0x44,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_sbyte v2, off, off ; encoding: [0x00,0x40,0x44,0xdc,0x00,0x00,0x7f,0x02]
+0x00,0x40,0x44,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sbyte v2, off, off offset:16 ; encoding: [0x10,0x40,0x44,0xdc,0x00,0x00,0x7f,0x02]
+0x10,0x40,0x44,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sbyte_d16 a2, v4, s6 ; encoding: [0x00,0x60,0x88,0xdc,0x04,0x00,0x86,0x02]
+0x00,0x60,0x88,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_sbyte_d16 a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x88,0xdc,0x04,0x00,0x86,0x02]
+0x10,0x60,0x88,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_sbyte_d16 a2, v4, off ; encoding: [0x00,0x60,0x88,0xdc,0x04,0x00,0xff,0x02]
+0x00,0x60,0x88,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_sbyte_d16 a2, v4, off offset:16 ; encoding: [0x10,0x60,0x88,0xdc,0x04,0x00,0xff,0x02]
+0x10,0x60,0x88,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_sbyte_d16 a2, off, s6 ; encoding: [0x00,0x40,0x88,0xdc,0x00,0x00,0x86,0x02]
+0x00,0x40,0x88,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_sbyte_d16 a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x88,0xdc,0x00,0x00,0x86,0x02]
+0x10,0x40,0x88,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_sbyte_d16 a2, off, off ; encoding: [0x00,0x40,0x88,0xdc,0x00,0x00,0xff,0x02]
+0x00,0x40,0x88,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_sbyte_d16 a2, off, off offset:16 ; encoding: [0x10,0x40,0x88,0xdc,0x00,0x00,0xff,0x02]
+0x10,0x40,0x88,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_sbyte_d16 v2, v4, s6 ; encoding: [0x00,0x60,0x88,0xdc,0x04,0x00,0x06,0x02]
+0x00,0x60,0x88,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_sbyte_d16 v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x88,0xdc,0x04,0x00,0x06,0x02]
+0x10,0x60,0x88,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_sbyte_d16 v2, v4, off ; encoding: [0x00,0x60,0x88,0xdc,0x04,0x00,0x7f,0x02]
+0x00,0x60,0x88,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sbyte_d16 v2, v4, off offset:16 ; encoding: [0x10,0x60,0x88,0xdc,0x04,0x00,0x7f,0x02]
+0x10,0x60,0x88,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sbyte_d16 v2, off, s6 ; encoding: [0x00,0x40,0x88,0xdc,0x00,0x00,0x06,0x02]
+0x00,0x40,0x88,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_sbyte_d16 v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x88,0xdc,0x00,0x00,0x06,0x02]
+0x10,0x40,0x88,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_sbyte_d16 v2, off, off ; encoding: [0x00,0x40,0x88,0xdc,0x00,0x00,0x7f,0x02]
+0x00,0x40,0x88,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sbyte_d16 v2, off, off offset:16 ; encoding: [0x10,0x40,0x88,0xdc,0x00,0x00,0x7f,0x02]
+0x10,0x40,0x88,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi a2, v4, s6 ; encoding: [0x00,0x60,0x8c,0xdc,0x04,0x00,0x86,0x02]
+0x00,0x60,0x8c,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x8c,0xdc,0x04,0x00,0x86,0x02]
+0x10,0x60,0x8c,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi a2, v4, off ; encoding: [0x00,0x60,0x8c,0xdc,0x04,0x00,0xff,0x02]
+0x00,0x60,0x8c,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi a2, v4, off offset:16 ; encoding: [0x10,0x60,0x8c,0xdc,0x04,0x00,0xff,0x02]
+0x10,0x60,0x8c,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi a2, off, s6 ; encoding: [0x00,0x40,0x8c,0xdc,0x00,0x00,0x86,0x02]
+0x00,0x40,0x8c,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x8c,0xdc,0x00,0x00,0x86,0x02]
+0x10,0x40,0x8c,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi a2, off, off ; encoding: [0x00,0x40,0x8c,0xdc,0x00,0x00,0xff,0x02]
+0x00,0x40,0x8c,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi a2, off, off offset:16 ; encoding: [0x10,0x40,0x8c,0xdc,0x00,0x00,0xff,0x02]
+0x10,0x40,0x8c,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi v2, v4, s6 ; encoding: [0x00,0x60,0x8c,0xdc,0x04,0x00,0x06,0x02]
+0x00,0x60,0x8c,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x8c,0xdc,0x04,0x00,0x06,0x02]
+0x10,0x60,0x8c,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi v2, v4, off ; encoding: [0x00,0x60,0x8c,0xdc,0x04,0x00,0x7f,0x02]
+0x00,0x60,0x8c,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi v2, v4, off offset:16 ; encoding: [0x10,0x60,0x8c,0xdc,0x04,0x00,0x7f,0x02]
+0x10,0x60,0x8c,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi v2, off, s6 ; encoding: [0x00,0x40,0x8c,0xdc,0x00,0x00,0x06,0x02]
+0x00,0x40,0x8c,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x8c,0xdc,0x00,0x00,0x06,0x02]
+0x10,0x40,0x8c,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi v2, off, off ; encoding: [0x00,0x40,0x8c,0xdc,0x00,0x00,0x7f,0x02]
+0x00,0x40,0x8c,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sbyte_d16_hi v2, off, off offset:16 ; encoding: [0x10,0x40,0x8c,0xdc,0x00,0x00,0x7f,0x02]
+0x10,0x40,0x8c,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_short_d16 a2, v4, s6 ; encoding: [0x00,0x60,0x90,0xdc,0x04,0x00,0x86,0x02]
+0x00,0x60,0x90,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_short_d16 a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x90,0xdc,0x04,0x00,0x86,0x02]
+0x10,0x60,0x90,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_short_d16 a2, v4, off ; encoding: [0x00,0x60,0x90,0xdc,0x04,0x00,0xff,0x02]
+0x00,0x60,0x90,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_short_d16 a2, v4, off offset:16 ; encoding: [0x10,0x60,0x90,0xdc,0x04,0x00,0xff,0x02]
+0x10,0x60,0x90,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_short_d16 a2, off, s6 ; encoding: [0x00,0x40,0x90,0xdc,0x00,0x00,0x86,0x02]
+0x00,0x40,0x90,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_short_d16 a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x90,0xdc,0x00,0x00,0x86,0x02]
+0x10,0x40,0x90,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_short_d16 a2, off, off ; encoding: [0x00,0x40,0x90,0xdc,0x00,0x00,0xff,0x02]
+0x00,0x40,0x90,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_short_d16 a2, off, off offset:16 ; encoding: [0x10,0x40,0x90,0xdc,0x00,0x00,0xff,0x02]
+0x10,0x40,0x90,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_short_d16 v2, v4, s6 ; encoding: [0x00,0x60,0x90,0xdc,0x04,0x00,0x06,0x02]
+0x00,0x60,0x90,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_short_d16 v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x90,0xdc,0x04,0x00,0x06,0x02]
+0x10,0x60,0x90,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_short_d16 v2, v4, off ; encoding: [0x00,0x60,0x90,0xdc,0x04,0x00,0x7f,0x02]
+0x00,0x60,0x90,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_short_d16 v2, v4, off offset:16 ; encoding: [0x10,0x60,0x90,0xdc,0x04,0x00,0x7f,0x02]
+0x10,0x60,0x90,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_short_d16 v2, off, s6 ; encoding: [0x00,0x40,0x90,0xdc,0x00,0x00,0x06,0x02]
+0x00,0x40,0x90,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_short_d16 v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x90,0xdc,0x00,0x00,0x06,0x02]
+0x10,0x40,0x90,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_short_d16 v2, off, off ; encoding: [0x00,0x40,0x90,0xdc,0x00,0x00,0x7f,0x02]
+0x00,0x40,0x90,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_short_d16 v2, off, off offset:16 ; encoding: [0x10,0x40,0x90,0xdc,0x00,0x00,0x7f,0x02]
+0x10,0x40,0x90,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_short_d16_hi a2, v4, s6 ; encoding: [0x00,0x60,0x94,0xdc,0x04,0x00,0x86,0x02]
+0x00,0x60,0x94,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_short_d16_hi a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x94,0xdc,0x04,0x00,0x86,0x02]
+0x10,0x60,0x94,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_short_d16_hi a2, v4, off ; encoding: [0x00,0x60,0x94,0xdc,0x04,0x00,0xff,0x02]
+0x00,0x60,0x94,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_short_d16_hi a2, v4, off offset:16 ; encoding: [0x10,0x60,0x94,0xdc,0x04,0x00,0xff,0x02]
+0x10,0x60,0x94,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_short_d16_hi a2, off, s6 ; encoding: [0x00,0x40,0x94,0xdc,0x00,0x00,0x86,0x02]
+0x00,0x40,0x94,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_short_d16_hi a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x94,0xdc,0x00,0x00,0x86,0x02]
+0x10,0x40,0x94,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_short_d16_hi a2, off, off ; encoding: [0x00,0x40,0x94,0xdc,0x00,0x00,0xff,0x02]
+0x00,0x40,0x94,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_short_d16_hi a2, off, off offset:16 ; encoding: [0x10,0x40,0x94,0xdc,0x00,0x00,0xff,0x02]
+0x10,0x40,0x94,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_short_d16_hi v2, v4, s6 ; encoding: [0x00,0x60,0x94,0xdc,0x04,0x00,0x06,0x02]
+0x00,0x60,0x94,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_short_d16_hi v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x94,0xdc,0x04,0x00,0x06,0x02]
+0x10,0x60,0x94,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_short_d16_hi v2, v4, off ; encoding: [0x00,0x60,0x94,0xdc,0x04,0x00,0x7f,0x02]
+0x00,0x60,0x94,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_short_d16_hi v2, v4, off offset:16 ; encoding: [0x10,0x60,0x94,0xdc,0x04,0x00,0x7f,0x02]
+0x10,0x60,0x94,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_short_d16_hi v2, off, s6 ; encoding: [0x00,0x40,0x94,0xdc,0x00,0x00,0x06,0x02]
+0x00,0x40,0x94,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_short_d16_hi v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x94,0xdc,0x00,0x00,0x06,0x02]
+0x10,0x40,0x94,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_short_d16_hi v2, off, off ; encoding: [0x00,0x40,0x94,0xdc,0x00,0x00,0x7f,0x02]
+0x00,0x40,0x94,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_short_d16_hi v2, off, off offset:16 ; encoding: [0x10,0x40,0x94,0xdc,0x00,0x00,0x7f,0x02]
+0x10,0x40,0x94,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sshort a2, v4, s6 ; encoding: [0x00,0x60,0x4c,0xdc,0x04,0x00,0x86,0x02]
+0x00,0x60,0x4c,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_sshort a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x4c,0xdc,0x04,0x00,0x86,0x02]
+0x10,0x60,0x4c,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_sshort a2, v4, off ; encoding: [0x00,0x60,0x4c,0xdc,0x04,0x00,0xff,0x02]
+0x00,0x60,0x4c,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_sshort a2, v4, off offset:16 ; encoding: [0x10,0x60,0x4c,0xdc,0x04,0x00,0xff,0x02]
+0x10,0x60,0x4c,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_sshort a2, off, s6 ; encoding: [0x00,0x40,0x4c,0xdc,0x00,0x00,0x86,0x02]
+0x00,0x40,0x4c,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_sshort a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x4c,0xdc,0x00,0x00,0x86,0x02]
+0x10,0x40,0x4c,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_sshort a2, off, off ; encoding: [0x00,0x40,0x4c,0xdc,0x00,0x00,0xff,0x02]
+0x00,0x40,0x4c,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_sshort a2, off, off offset:16 ; encoding: [0x10,0x40,0x4c,0xdc,0x00,0x00,0xff,0x02]
+0x10,0x40,0x4c,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_sshort v2, v4, s6 ; encoding: [0x00,0x60,0x4c,0xdc,0x04,0x00,0x06,0x02]
+0x00,0x60,0x4c,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_sshort v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x4c,0xdc,0x04,0x00,0x06,0x02]
+0x10,0x60,0x4c,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_sshort v2, v4, off ; encoding: [0x00,0x60,0x4c,0xdc,0x04,0x00,0x7f,0x02]
+0x00,0x60,0x4c,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sshort v2, v4, off offset:16 ; encoding: [0x10,0x60,0x4c,0xdc,0x04,0x00,0x7f,0x02]
+0x10,0x60,0x4c,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sshort v2, off, s6 ; encoding: [0x00,0x40,0x4c,0xdc,0x00,0x00,0x06,0x02]
+0x00,0x40,0x4c,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_sshort v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x4c,0xdc,0x00,0x00,0x06,0x02]
+0x10,0x40,0x4c,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_sshort v2, off, off ; encoding: [0x00,0x40,0x4c,0xdc,0x00,0x00,0x7f,0x02]
+0x00,0x40,0x4c,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_sshort v2, off, off offset:16 ; encoding: [0x10,0x40,0x4c,0xdc,0x00,0x00,0x7f,0x02]
+0x10,0x40,0x4c,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ubyte a2, v4, s6 ; encoding: [0x00,0x60,0x40,0xdc,0x04,0x00,0x86,0x02]
+0x00,0x60,0x40,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_ubyte a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x40,0xdc,0x04,0x00,0x86,0x02]
+0x10,0x60,0x40,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_ubyte a2, v4, off ; encoding: [0x00,0x60,0x40,0xdc,0x04,0x00,0xff,0x02]
+0x00,0x60,0x40,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_ubyte a2, v4, off offset:16 ; encoding: [0x10,0x60,0x40,0xdc,0x04,0x00,0xff,0x02]
+0x10,0x60,0x40,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_ubyte a2, off, s6 ; encoding: [0x00,0x40,0x40,0xdc,0x00,0x00,0x86,0x02]
+0x00,0x40,0x40,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_ubyte a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x40,0xdc,0x00,0x00,0x86,0x02]
+0x10,0x40,0x40,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_ubyte a2, off, off ; encoding: [0x00,0x40,0x40,0xdc,0x00,0x00,0xff,0x02]
+0x00,0x40,0x40,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_ubyte a2, off, off offset:16 ; encoding: [0x10,0x40,0x40,0xdc,0x00,0x00,0xff,0x02]
+0x10,0x40,0x40,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_ubyte v2, v4, s6 ; encoding: [0x00,0x60,0x40,0xdc,0x04,0x00,0x06,0x02]
+0x00,0x60,0x40,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_ubyte v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x40,0xdc,0x04,0x00,0x06,0x02]
+0x10,0x60,0x40,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_ubyte v2, v4, off ; encoding: [0x00,0x60,0x40,0xdc,0x04,0x00,0x7f,0x02]
+0x00,0x60,0x40,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ubyte v2, v4, off offset:16 ; encoding: [0x10,0x60,0x40,0xdc,0x04,0x00,0x7f,0x02]
+0x10,0x60,0x40,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ubyte v2, off, s6 ; encoding: [0x00,0x40,0x40,0xdc,0x00,0x00,0x06,0x02]
+0x00,0x40,0x40,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_ubyte v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x40,0xdc,0x00,0x00,0x06,0x02]
+0x10,0x40,0x40,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_ubyte v2, off, off ; encoding: [0x00,0x40,0x40,0xdc,0x00,0x00,0x7f,0x02]
+0x00,0x40,0x40,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ubyte v2, off, off offset:16 ; encoding: [0x10,0x40,0x40,0xdc,0x00,0x00,0x7f,0x02]
+0x10,0x40,0x40,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ubyte_d16 a2, v4, s6 ; encoding: [0x00,0x60,0x80,0xdc,0x04,0x00,0x86,0x02]
+0x00,0x60,0x80,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_ubyte_d16 a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x80,0xdc,0x04,0x00,0x86,0x02]
+0x10,0x60,0x80,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_ubyte_d16 a2, v4, off ; encoding: [0x00,0x60,0x80,0xdc,0x04,0x00,0xff,0x02]
+0x00,0x60,0x80,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_ubyte_d16 a2, v4, off offset:16 ; encoding: [0x10,0x60,0x80,0xdc,0x04,0x00,0xff,0x02]
+0x10,0x60,0x80,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_ubyte_d16 a2, off, s6 ; encoding: [0x00,0x40,0x80,0xdc,0x00,0x00,0x86,0x02]
+0x00,0x40,0x80,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_ubyte_d16 a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x80,0xdc,0x00,0x00,0x86,0x02]
+0x10,0x40,0x80,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_ubyte_d16 a2, off, off ; encoding: [0x00,0x40,0x80,0xdc,0x00,0x00,0xff,0x02]
+0x00,0x40,0x80,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_ubyte_d16 a2, off, off offset:16 ; encoding: [0x10,0x40,0x80,0xdc,0x00,0x00,0xff,0x02]
+0x10,0x40,0x80,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_ubyte_d16 v2, v4, s6 ; encoding: [0x00,0x60,0x80,0xdc,0x04,0x00,0x06,0x02]
+0x00,0x60,0x80,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_ubyte_d16 v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x80,0xdc,0x04,0x00,0x06,0x02]
+0x10,0x60,0x80,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_ubyte_d16 v2, v4, off ; encoding: [0x00,0x60,0x80,0xdc,0x04,0x00,0x7f,0x02]
+0x00,0x60,0x80,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ubyte_d16 v2, v4, off offset:16 ; encoding: [0x10,0x60,0x80,0xdc,0x04,0x00,0x7f,0x02]
+0x10,0x60,0x80,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ubyte_d16 v2, off, s6 ; encoding: [0x00,0x40,0x80,0xdc,0x00,0x00,0x06,0x02]
+0x00,0x40,0x80,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_ubyte_d16 v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x80,0xdc,0x00,0x00,0x06,0x02]
+0x10,0x40,0x80,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_ubyte_d16 v2, off, off ; encoding: [0x00,0x40,0x80,0xdc,0x00,0x00,0x7f,0x02]
+0x00,0x40,0x80,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ubyte_d16 v2, off, off offset:16 ; encoding: [0x10,0x40,0x80,0xdc,0x00,0x00,0x7f,0x02]
+0x10,0x40,0x80,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi a2, v4, s6 ; encoding: [0x00,0x60,0x84,0xdc,0x04,0x00,0x86,0x02]
+0x00,0x60,0x84,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x84,0xdc,0x04,0x00,0x86,0x02]
+0x10,0x60,0x84,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi a2, v4, off ; encoding: [0x00,0x60,0x84,0xdc,0x04,0x00,0xff,0x02]
+0x00,0x60,0x84,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi a2, v4, off offset:16 ; encoding: [0x10,0x60,0x84,0xdc,0x04,0x00,0xff,0x02]
+0x10,0x60,0x84,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi a2, off, s6 ; encoding: [0x00,0x40,0x84,0xdc,0x00,0x00,0x86,0x02]
+0x00,0x40,0x84,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x84,0xdc,0x00,0x00,0x86,0x02]
+0x10,0x40,0x84,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi a2, off, off ; encoding: [0x00,0x40,0x84,0xdc,0x00,0x00,0xff,0x02]
+0x00,0x40,0x84,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi a2, off, off offset:16 ; encoding: [0x10,0x40,0x84,0xdc,0x00,0x00,0xff,0x02]
+0x10,0x40,0x84,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi v2, v4, s6 ; encoding: [0x00,0x60,0x84,0xdc,0x04,0x00,0x06,0x02]
+0x00,0x60,0x84,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x84,0xdc,0x04,0x00,0x06,0x02]
+0x10,0x60,0x84,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi v2, v4, off ; encoding: [0x00,0x60,0x84,0xdc,0x04,0x00,0x7f,0x02]
+0x00,0x60,0x84,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi v2, v4, off offset:16 ; encoding: [0x10,0x60,0x84,0xdc,0x04,0x00,0x7f,0x02]
+0x10,0x60,0x84,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi v2, off, s6 ; encoding: [0x00,0x40,0x84,0xdc,0x00,0x00,0x06,0x02]
+0x00,0x40,0x84,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x84,0xdc,0x00,0x00,0x06,0x02]
+0x10,0x40,0x84,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi v2, off, off ; encoding: [0x00,0x40,0x84,0xdc,0x00,0x00,0x7f,0x02]
+0x00,0x40,0x84,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ubyte_d16_hi v2, off, off offset:16 ; encoding: [0x10,0x40,0x84,0xdc,0x00,0x00,0x7f,0x02]
+0x10,0x40,0x84,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ushort a2, v4, s6 ; encoding: [0x00,0x60,0x48,0xdc,0x04,0x00,0x86,0x02]
+0x00,0x60,0x48,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_ushort a2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x48,0xdc,0x04,0x00,0x86,0x02]
+0x10,0x60,0x48,0xdc,0x04,0x00,0x86,0x02
+
+# GFX940: scratch_load_ushort a2, v4, off ; encoding: [0x00,0x60,0x48,0xdc,0x04,0x00,0xff,0x02]
+0x00,0x60,0x48,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_ushort a2, v4, off offset:16 ; encoding: [0x10,0x60,0x48,0xdc,0x04,0x00,0xff,0x02]
+0x10,0x60,0x48,0xdc,0x04,0x00,0xff,0x02
+
+# GFX940: scratch_load_ushort a2, off, s6 ; encoding: [0x00,0x40,0x48,0xdc,0x00,0x00,0x86,0x02]
+0x00,0x40,0x48,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_ushort a2, off, s6 offset:16 ; encoding: [0x10,0x40,0x48,0xdc,0x00,0x00,0x86,0x02]
+0x10,0x40,0x48,0xdc,0x00,0x00,0x86,0x02
+
+# GFX940: scratch_load_ushort a2, off, off ; encoding: [0x00,0x40,0x48,0xdc,0x00,0x00,0xff,0x02]
+0x00,0x40,0x48,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_ushort a2, off, off offset:16 ; encoding: [0x10,0x40,0x48,0xdc,0x00,0x00,0xff,0x02]
+0x10,0x40,0x48,0xdc,0x00,0x00,0xff,0x02
+
+# GFX940: scratch_load_ushort v2, v4, s6 ; encoding: [0x00,0x60,0x48,0xdc,0x04,0x00,0x06,0x02]
+0x00,0x60,0x48,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_ushort v2, v4, s6 offset:16 ; encoding: [0x10,0x60,0x48,0xdc,0x04,0x00,0x06,0x02]
+0x10,0x60,0x48,0xdc,0x04,0x00,0x06,0x02
+
+# GFX940: scratch_load_ushort v2, v4, off ; encoding: [0x00,0x60,0x48,0xdc,0x04,0x00,0x7f,0x02]
+0x00,0x60,0x48,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ushort v2, v4, off offset:16 ; encoding: [0x10,0x60,0x48,0xdc,0x04,0x00,0x7f,0x02]
+0x10,0x60,0x48,0xdc,0x04,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ushort v2, off, s6 ; encoding: [0x00,0x40,0x48,0xdc,0x00,0x00,0x06,0x02]
+0x00,0x40,0x48,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_ushort v2, off, s6 offset:16 ; encoding: [0x10,0x40,0x48,0xdc,0x00,0x00,0x06,0x02]
+0x10,0x40,0x48,0xdc,0x00,0x00,0x06,0x02
+
+# GFX940: scratch_load_ushort v2, off, off ; encoding: [0x00,0x40,0x48,0xdc,0x00,0x00,0x7f,0x02]
+0x00,0x40,0x48,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_load_ushort v2, off, off offset:16 ; encoding: [0x10,0x40,0x48,0xdc,0x00,0x00,0x7f,0x02]
+0x10,0x40,0x48,0xdc,0x00,0x00,0x7f,0x02
+
+# GFX940: scratch_store_byte v4, a2, s6 ; encoding: [0x00,0x60,0x60,0xdc,0x04,0x02,0x86,0x00]
+0x00,0x60,0x60,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_byte v4, a2, s6 offset:16 ; encoding: [0x10,0x60,0x60,0xdc,0x04,0x02,0x86,0x00]
+0x10,0x60,0x60,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_byte v4, a2, off ; encoding: [0x00,0x60,0x60,0xdc,0x04,0x02,0xff,0x00]
+0x00,0x60,0x60,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_byte v4, a2, off offset:16 ; encoding: [0x10,0x60,0x60,0xdc,0x04,0x02,0xff,0x00]
+0x10,0x60,0x60,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_byte off, a2, s6 ; encoding: [0x00,0x40,0x60,0xdc,0x00,0x02,0x86,0x00]
+0x00,0x40,0x60,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_byte off, a2, s6 offset:16 ; encoding: [0x10,0x40,0x60,0xdc,0x00,0x02,0x86,0x00]
+0x10,0x40,0x60,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_byte off, a2, off ; encoding: [0x00,0x40,0x60,0xdc,0x00,0x02,0xff,0x00]
+0x00,0x40,0x60,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_byte off, a2, off offset:16 ; encoding: [0x10,0x40,0x60,0xdc,0x00,0x02,0xff,0x00]
+0x10,0x40,0x60,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_byte v4, v2, s6 ; encoding: [0x00,0x60,0x60,0xdc,0x04,0x02,0x06,0x00]
+0x00,0x60,0x60,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_byte v4, v2, s6 offset:16 ; encoding: [0x10,0x60,0x60,0xdc,0x04,0x02,0x06,0x00]
+0x10,0x60,0x60,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_byte v4, v2, off ; encoding: [0x00,0x60,0x60,0xdc,0x04,0x02,0x7f,0x00]
+0x00,0x60,0x60,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_byte v4, v2, off offset:16 ; encoding: [0x10,0x60,0x60,0xdc,0x04,0x02,0x7f,0x00]
+0x10,0x60,0x60,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_byte off, v2, s6 ; encoding: [0x00,0x40,0x60,0xdc,0x00,0x02,0x06,0x00]
+0x00,0x40,0x60,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_byte off, v2, s6 offset:16 ; encoding: [0x10,0x40,0x60,0xdc,0x00,0x02,0x06,0x00]
+0x10,0x40,0x60,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_byte off, v2, off ; encoding: [0x00,0x40,0x60,0xdc,0x00,0x02,0x7f,0x00]
+0x00,0x40,0x60,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_byte off, v2, off offset:16 ; encoding: [0x10,0x40,0x60,0xdc,0x00,0x02,0x7f,0x00]
+0x10,0x40,0x60,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_byte_d16_hi v4, a2, s6 ; encoding: [0x00,0x60,0x64,0xdc,0x04,0x02,0x86,0x00]
+0x00,0x60,0x64,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_byte_d16_hi v4, a2, s6 offset:16 ; encoding: [0x10,0x60,0x64,0xdc,0x04,0x02,0x86,0x00]
+0x10,0x60,0x64,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_byte_d16_hi v4, a2, off ; encoding: [0x00,0x60,0x64,0xdc,0x04,0x02,0xff,0x00]
+0x00,0x60,0x64,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_byte_d16_hi v4, a2, off offset:16 ; encoding: [0x10,0x60,0x64,0xdc,0x04,0x02,0xff,0x00]
+0x10,0x60,0x64,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_byte_d16_hi off, a2, s6 ; encoding: [0x00,0x40,0x64,0xdc,0x00,0x02,0x86,0x00]
+0x00,0x40,0x64,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_byte_d16_hi off, a2, s6 offset:16 ; encoding: [0x10,0x40,0x64,0xdc,0x00,0x02,0x86,0x00]
+0x10,0x40,0x64,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_byte_d16_hi off, a2, off ; encoding: [0x00,0x40,0x64,0xdc,0x00,0x02,0xff,0x00]
+0x00,0x40,0x64,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_byte_d16_hi off, a2, off offset:16 ; encoding: [0x10,0x40,0x64,0xdc,0x00,0x02,0xff,0x00]
+0x10,0x40,0x64,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_byte_d16_hi v4, v2, s6 ; encoding: [0x00,0x60,0x64,0xdc,0x04,0x02,0x06,0x00]
+0x00,0x60,0x64,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_byte_d16_hi v4, v2, s6 offset:16 ; encoding: [0x10,0x60,0x64,0xdc,0x04,0x02,0x06,0x00]
+0x10,0x60,0x64,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_byte_d16_hi v4, v2, off ; encoding: [0x00,0x60,0x64,0xdc,0x04,0x02,0x7f,0x00]
+0x00,0x60,0x64,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_byte_d16_hi v4, v2, off offset:16 ; encoding: [0x10,0x60,0x64,0xdc,0x04,0x02,0x7f,0x00]
+0x10,0x60,0x64,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_byte_d16_hi off, v2, s6 ; encoding: [0x00,0x40,0x64,0xdc,0x00,0x02,0x06,0x00]
+0x00,0x40,0x64,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_byte_d16_hi off, v2, s6 offset:16 ; encoding: [0x10,0x40,0x64,0xdc,0x00,0x02,0x06,0x00]
+0x10,0x40,0x64,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_byte_d16_hi off, v2, off ; encoding: [0x00,0x40,0x64,0xdc,0x00,0x02,0x7f,0x00]
+0x00,0x40,0x64,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_byte_d16_hi off, v2, off offset:16 ; encoding: [0x10,0x40,0x64,0xdc,0x00,0x02,0x7f,0x00]
+0x10,0x40,0x64,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dword v4, a2, s6 ; encoding: [0x00,0x60,0x70,0xdc,0x04,0x02,0x86,0x00]
+0x00,0x60,0x70,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_dword v4, a2, s6 offset:16 ; encoding: [0x10,0x60,0x70,0xdc,0x04,0x02,0x86,0x00]
+0x10,0x60,0x70,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_dword v4, a2, off ; encoding: [0x00,0x60,0x70,0xdc,0x04,0x02,0xff,0x00]
+0x00,0x60,0x70,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_dword v4, a2, off offset:16 ; encoding: [0x10,0x60,0x70,0xdc,0x04,0x02,0xff,0x00]
+0x10,0x60,0x70,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_dword off, a2, s6 ; encoding: [0x00,0x40,0x70,0xdc,0x00,0x02,0x86,0x00]
+0x00,0x40,0x70,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_dword off, a2, s6 offset:16 ; encoding: [0x10,0x40,0x70,0xdc,0x00,0x02,0x86,0x00]
+0x10,0x40,0x70,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_dword off, a2, off ; encoding: [0x00,0x40,0x70,0xdc,0x00,0x02,0xff,0x00]
+0x00,0x40,0x70,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_dword off, a2, off offset:16 ; encoding: [0x10,0x40,0x70,0xdc,0x00,0x02,0xff,0x00]
+0x10,0x40,0x70,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_dword v4, v2, s6 ; encoding: [0x00,0x60,0x70,0xdc,0x04,0x02,0x06,0x00]
+0x00,0x60,0x70,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_dword v4, v2, s6 offset:16 ; encoding: [0x10,0x60,0x70,0xdc,0x04,0x02,0x06,0x00]
+0x10,0x60,0x70,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_dword v4, v2, off ; encoding: [0x00,0x60,0x70,0xdc,0x04,0x02,0x7f,0x00]
+0x00,0x60,0x70,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dword v4, v2, off offset:16 ; encoding: [0x10,0x60,0x70,0xdc,0x04,0x02,0x7f,0x00]
+0x10,0x60,0x70,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dword off, v2, s6 ; encoding: [0x00,0x40,0x70,0xdc,0x00,0x02,0x06,0x00]
+0x00,0x40,0x70,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_dword off, v2, s6 offset:16 ; encoding: [0x10,0x40,0x70,0xdc,0x00,0x02,0x06,0x00]
+0x10,0x40,0x70,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_dword off, v2, off ; encoding: [0x00,0x40,0x70,0xdc,0x00,0x02,0x7f,0x00]
+0x00,0x40,0x70,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dword off, v2, off offset:16 ; encoding: [0x10,0x40,0x70,0xdc,0x00,0x02,0x7f,0x00]
+0x10,0x40,0x70,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dwordx2 v4, a[2:3], s6 ; encoding: [0x00,0x60,0x74,0xdc,0x04,0x02,0x86,0x00]
+0x00,0x60,0x74,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_dwordx2 v4, a[2:3], s6 offset:16 ; encoding: [0x10,0x60,0x74,0xdc,0x04,0x02,0x86,0x00]
+0x10,0x60,0x74,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_dwordx2 v4, a[2:3], off ; encoding: [0x00,0x60,0x74,0xdc,0x04,0x02,0xff,0x00]
+0x00,0x60,0x74,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_dwordx2 v4, a[2:3], off offset:16 ; encoding: [0x10,0x60,0x74,0xdc,0x04,0x02,0xff,0x00]
+0x10,0x60,0x74,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_dwordx2 off, a[2:3], s6 ; encoding: [0x00,0x40,0x74,0xdc,0x00,0x02,0x86,0x00]
+0x00,0x40,0x74,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_dwordx2 off, a[2:3], s6 offset:16 ; encoding: [0x10,0x40,0x74,0xdc,0x00,0x02,0x86,0x00]
+0x10,0x40,0x74,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_dwordx2 off, a[2:3], off ; encoding: [0x00,0x40,0x74,0xdc,0x00,0x02,0xff,0x00]
+0x00,0x40,0x74,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_dwordx2 off, a[2:3], off offset:16 ; encoding: [0x10,0x40,0x74,0xdc,0x00,0x02,0xff,0x00]
+0x10,0x40,0x74,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_dwordx2 v4, v[2:3], s6 ; encoding: [0x00,0x60,0x74,0xdc,0x04,0x02,0x06,0x00]
+0x00,0x60,0x74,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_dwordx2 v4, v[2:3], s6 offset:16 ; encoding: [0x10,0x60,0x74,0xdc,0x04,0x02,0x06,0x00]
+0x10,0x60,0x74,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_dwordx2 v4, v[2:3], off ; encoding: [0x00,0x60,0x74,0xdc,0x04,0x02,0x7f,0x00]
+0x00,0x60,0x74,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dwordx2 v4, v[2:3], off offset:16 ; encoding: [0x10,0x60,0x74,0xdc,0x04,0x02,0x7f,0x00]
+0x10,0x60,0x74,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dwordx2 off, v[2:3], s6 ; encoding: [0x00,0x40,0x74,0xdc,0x00,0x02,0x06,0x00]
+0x00,0x40,0x74,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_dwordx2 off, v[2:3], s6 offset:16 ; encoding: [0x10,0x40,0x74,0xdc,0x00,0x02,0x06,0x00]
+0x10,0x40,0x74,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_dwordx2 off, v[2:3], off ; encoding: [0x00,0x40,0x74,0xdc,0x00,0x02,0x7f,0x00]
+0x00,0x40,0x74,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dwordx2 off, v[2:3], off offset:16 ; encoding: [0x10,0x40,0x74,0xdc,0x00,0x02,0x7f,0x00]
+0x10,0x40,0x74,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dwordx3 v4, a[2:4], s6 ; encoding: [0x00,0x60,0x78,0xdc,0x04,0x02,0x86,0x00]
+0x00,0x60,0x78,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_dwordx3 v4, a[2:4], s6 offset:16 ; encoding: [0x10,0x60,0x78,0xdc,0x04,0x02,0x86,0x00]
+0x10,0x60,0x78,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_dwordx3 v4, a[2:4], off ; encoding: [0x00,0x60,0x78,0xdc,0x04,0x02,0xff,0x00]
+0x00,0x60,0x78,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_dwordx3 v4, a[2:4], off offset:16 ; encoding: [0x10,0x60,0x78,0xdc,0x04,0x02,0xff,0x00]
+0x10,0x60,0x78,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_dwordx3 off, a[2:4], s6 ; encoding: [0x00,0x40,0x78,0xdc,0x00,0x02,0x86,0x00]
+0x00,0x40,0x78,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_dwordx3 off, a[2:4], s6 offset:16 ; encoding: [0x10,0x40,0x78,0xdc,0x00,0x02,0x86,0x00]
+0x10,0x40,0x78,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_dwordx3 off, a[2:4], off ; encoding: [0x00,0x40,0x78,0xdc,0x00,0x02,0xff,0x00]
+0x00,0x40,0x78,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_dwordx3 off, a[2:4], off offset:16 ; encoding: [0x10,0x40,0x78,0xdc,0x00,0x02,0xff,0x00]
+0x10,0x40,0x78,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_dwordx3 v4, v[2:4], s6 ; encoding: [0x00,0x60,0x78,0xdc,0x04,0x02,0x06,0x00]
+0x00,0x60,0x78,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_dwordx3 v4, v[2:4], s6 offset:16 ; encoding: [0x10,0x60,0x78,0xdc,0x04,0x02,0x06,0x00]
+0x10,0x60,0x78,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_dwordx3 v4, v[2:4], off ; encoding: [0x00,0x60,0x78,0xdc,0x04,0x02,0x7f,0x00]
+0x00,0x60,0x78,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dwordx3 v4, v[2:4], off offset:16 ; encoding: [0x10,0x60,0x78,0xdc,0x04,0x02,0x7f,0x00]
+0x10,0x60,0x78,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dwordx3 off, v[2:4], s6 ; encoding: [0x00,0x40,0x78,0xdc,0x00,0x02,0x06,0x00]
+0x00,0x40,0x78,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_dwordx3 off, v[2:4], s6 offset:16 ; encoding: [0x10,0x40,0x78,0xdc,0x00,0x02,0x06,0x00]
+0x10,0x40,0x78,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_dwordx3 off, v[2:4], off ; encoding: [0x00,0x40,0x78,0xdc,0x00,0x02,0x7f,0x00]
+0x00,0x40,0x78,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dwordx3 off, v[2:4], off offset:16 ; encoding: [0x10,0x40,0x78,0xdc,0x00,0x02,0x7f,0x00]
+0x10,0x40,0x78,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dwordx4 v4, a[2:5], s6 ; encoding: [0x00,0x60,0x7c,0xdc,0x04,0x02,0x86,0x00]
+0x00,0x60,0x7c,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_dwordx4 v4, a[2:5], s6 offset:16 ; encoding: [0x10,0x60,0x7c,0xdc,0x04,0x02,0x86,0x00]
+0x10,0x60,0x7c,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_dwordx4 v4, a[2:5], off ; encoding: [0x00,0x60,0x7c,0xdc,0x04,0x02,0xff,0x00]
+0x00,0x60,0x7c,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_dwordx4 v4, a[2:5], off offset:16 ; encoding: [0x10,0x60,0x7c,0xdc,0x04,0x02,0xff,0x00]
+0x10,0x60,0x7c,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_dwordx4 off, a[2:5], s6 ; encoding: [0x00,0x40,0x7c,0xdc,0x00,0x02,0x86,0x00]
+0x00,0x40,0x7c,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_dwordx4 off, a[2:5], s6 offset:16 ; encoding: [0x10,0x40,0x7c,0xdc,0x00,0x02,0x86,0x00]
+0x10,0x40,0x7c,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_dwordx4 off, a[2:5], off ; encoding: [0x00,0x40,0x7c,0xdc,0x00,0x02,0xff,0x00]
+0x00,0x40,0x7c,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_dwordx4 off, a[2:5], off offset:16 ; encoding: [0x10,0x40,0x7c,0xdc,0x00,0x02,0xff,0x00]
+0x10,0x40,0x7c,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_dwordx4 v4, v[2:5], s6 ; encoding: [0x00,0x60,0x7c,0xdc,0x04,0x02,0x06,0x00]
+0x00,0x60,0x7c,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_dwordx4 v4, v[2:5], s6 offset:16 ; encoding: [0x10,0x60,0x7c,0xdc,0x04,0x02,0x06,0x00]
+0x10,0x60,0x7c,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_dwordx4 v4, v[2:5], off ; encoding: [0x00,0x60,0x7c,0xdc,0x04,0x02,0x7f,0x00]
+0x00,0x60,0x7c,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dwordx4 v4, v[2:5], off offset:16 ; encoding: [0x10,0x60,0x7c,0xdc,0x04,0x02,0x7f,0x00]
+0x10,0x60,0x7c,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dwordx4 off, v[2:5], s6 ; encoding: [0x00,0x40,0x7c,0xdc,0x00,0x02,0x06,0x00]
+0x00,0x40,0x7c,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_dwordx4 off, v[2:5], s6 offset:16 ; encoding: [0x10,0x40,0x7c,0xdc,0x00,0x02,0x06,0x00]
+0x10,0x40,0x7c,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_dwordx4 off, v[2:5], off ; encoding: [0x00,0x40,0x7c,0xdc,0x00,0x02,0x7f,0x00]
+0x00,0x40,0x7c,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_dwordx4 off, v[2:5], off offset:16 ; encoding: [0x10,0x40,0x7c,0xdc,0x00,0x02,0x7f,0x00]
+0x10,0x40,0x7c,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_short v4, a2, s6 ; encoding: [0x00,0x60,0x68,0xdc,0x04,0x02,0x86,0x00]
+0x00,0x60,0x68,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_short v4, a2, s6 offset:16 ; encoding: [0x10,0x60,0x68,0xdc,0x04,0x02,0x86,0x00]
+0x10,0x60,0x68,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_short v4, a2, off ; encoding: [0x00,0x60,0x68,0xdc,0x04,0x02,0xff,0x00]
+0x00,0x60,0x68,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_short v4, a2, off offset:16 ; encoding: [0x10,0x60,0x68,0xdc,0x04,0x02,0xff,0x00]
+0x10,0x60,0x68,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_short off, a2, s6 ; encoding: [0x00,0x40,0x68,0xdc,0x00,0x02,0x86,0x00]
+0x00,0x40,0x68,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_short off, a2, s6 offset:16 ; encoding: [0x10,0x40,0x68,0xdc,0x00,0x02,0x86,0x00]
+0x10,0x40,0x68,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_short off, a2, off ; encoding: [0x00,0x40,0x68,0xdc,0x00,0x02,0xff,0x00]
+0x00,0x40,0x68,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_short off, a2, off offset:16 ; encoding: [0x10,0x40,0x68,0xdc,0x00,0x02,0xff,0x00]
+0x10,0x40,0x68,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_short v4, v2, s6 ; encoding: [0x00,0x60,0x68,0xdc,0x04,0x02,0x06,0x00]
+0x00,0x60,0x68,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_short v4, v2, s6 offset:16 ; encoding: [0x10,0x60,0x68,0xdc,0x04,0x02,0x06,0x00]
+0x10,0x60,0x68,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_short v4, v2, off ; encoding: [0x00,0x60,0x68,0xdc,0x04,0x02,0x7f,0x00]
+0x00,0x60,0x68,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_short v4, v2, off offset:16 ; encoding: [0x10,0x60,0x68,0xdc,0x04,0x02,0x7f,0x00]
+0x10,0x60,0x68,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_short off, v2, s6 ; encoding: [0x00,0x40,0x68,0xdc,0x00,0x02,0x06,0x00]
+0x00,0x40,0x68,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_short off, v2, s6 offset:16 ; encoding: [0x10,0x40,0x68,0xdc,0x00,0x02,0x06,0x00]
+0x10,0x40,0x68,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_short off, v2, off ; encoding: [0x00,0x40,0x68,0xdc,0x00,0x02,0x7f,0x00]
+0x00,0x40,0x68,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_short off, v2, off offset:16 ; encoding: [0x10,0x40,0x68,0xdc,0x00,0x02,0x7f,0x00]
+0x10,0x40,0x68,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_short_d16_hi v4, a2, s6 ; encoding: [0x00,0x60,0x6c,0xdc,0x04,0x02,0x86,0x00]
+0x00,0x60,0x6c,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_short_d16_hi v4, a2, s6 offset:16 ; encoding: [0x10,0x60,0x6c,0xdc,0x04,0x02,0x86,0x00]
+0x10,0x60,0x6c,0xdc,0x04,0x02,0x86,0x00
+
+# GFX940: scratch_store_short_d16_hi v4, a2, off ; encoding: [0x00,0x60,0x6c,0xdc,0x04,0x02,0xff,0x00]
+0x00,0x60,0x6c,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_short_d16_hi v4, a2, off offset:16 ; encoding: [0x10,0x60,0x6c,0xdc,0x04,0x02,0xff,0x00]
+0x10,0x60,0x6c,0xdc,0x04,0x02,0xff,0x00
+
+# GFX940: scratch_store_short_d16_hi off, a2, s6 ; encoding: [0x00,0x40,0x6c,0xdc,0x00,0x02,0x86,0x00]
+0x00,0x40,0x6c,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_short_d16_hi off, a2, s6 offset:16 ; encoding: [0x10,0x40,0x6c,0xdc,0x00,0x02,0x86,0x00]
+0x10,0x40,0x6c,0xdc,0x00,0x02,0x86,0x00
+
+# GFX940: scratch_store_short_d16_hi off, a2, off ; encoding: [0x00,0x40,0x6c,0xdc,0x00,0x02,0xff,0x00]
+0x00,0x40,0x6c,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_short_d16_hi off, a2, off offset:16 ; encoding: [0x10,0x40,0x6c,0xdc,0x00,0x02,0xff,0x00]
+0x10,0x40,0x6c,0xdc,0x00,0x02,0xff,0x00
+
+# GFX940: scratch_store_short_d16_hi v4, v2, s6 ; encoding: [0x00,0x60,0x6c,0xdc,0x04,0x02,0x06,0x00]
+0x00,0x60,0x6c,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_short_d16_hi v4, v2, s6 offset:16 ; encoding: [0x10,0x60,0x6c,0xdc,0x04,0x02,0x06,0x00]
+0x10,0x60,0x6c,0xdc,0x04,0x02,0x06,0x00
+
+# GFX940: scratch_store_short_d16_hi v4, v2, off ; encoding: [0x00,0x60,0x6c,0xdc,0x04,0x02,0x7f,0x00]
+0x00,0x60,0x6c,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_short_d16_hi v4, v2, off offset:16 ; encoding: [0x10,0x60,0x6c,0xdc,0x04,0x02,0x7f,0x00]
+0x10,0x60,0x6c,0xdc,0x04,0x02,0x7f,0x00
+
+# GFX940: scratch_store_short_d16_hi off, v2, s6 ; encoding: [0x00,0x40,0x6c,0xdc,0x00,0x02,0x06,0x00]
+0x00,0x40,0x6c,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_short_d16_hi off, v2, s6 offset:16 ; encoding: [0x10,0x40,0x6c,0xdc,0x00,0x02,0x06,0x00]
+0x10,0x40,0x6c,0xdc,0x00,0x02,0x06,0x00
+
+# GFX940: scratch_store_short_d16_hi off, v2, off ; encoding: [0x00,0x40,0x6c,0xdc,0x00,0x02,0x7f,0x00]
+0x00,0x40,0x6c,0xdc,0x00,0x02,0x7f,0x00
+
+# GFX940: scratch_store_short_d16_hi off, v2, off offset:16 ; encoding: [0x10,0x40,0x6c,0xdc,0x00,0x02,0x7f,0x00]
+0x10,0x40,0x6c,0xdc,0x00,0x02,0x7f,0x00