[AMDGPU][MC] Enabled 21-bit signed offsets for SMEM instructions

[dp]

Summary

Added support for 21-bit signed SMEM offsets (GFX9+).

[arsenm]

llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

3427–3441

I think we're missing the same checks in the MachineVerifier. Can you avoid spreading the hardcoded number of bits to multiple places?

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

153

IsBuffer

[dp]

Corrected issues found by Matt.

[dp]

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

1313–1315

This code should probably be corrected as well but I do not know if codegen is ready for this change.

[arsenm]

llvm/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp

369

This assert is redundant, the getImm below will assert anyway

372–373

Do we really need to explicitly clamp this here?

[dp]

llvm/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp

369

My intention was to state explicitly that an immediate is expected. But this is a matter of taste. I’ll remove the assert.

372–373

This is not strictly necessary. However AMD documentation states that unused bits in instruction encoding should be zero. An MC instruction may come directly from codegen, and I’m not sure it does SMEM offset validation. Should I replace this truncation with an assert?

[dp]

Removed getSMEMOffsetEncoding

[arsenm]

llvm/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp

372–373

I would assume codegen would catch this in a verifier, and the assembler verifier would prevent this from getting here, so this should probably be an assert

[dp]

Restored getSMEMOffsetEncoding; replaced offset truncation with an assert as suggested by Matt.

[arsenm]

LGTM with assert nit

llvm/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp

370

isUINt<20> would be clearer

llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

bool isSoppBrTarget() const {
     return isExpr() || isImm();
   }
 
   bool isSWaitCnt() const;
   bool isHwreg() const;
   bool isSendMsg() const;
   bool isSwizzle() const;
   bool isSMRDOffset8() const;
-  bool isSMRDOffset20() const;
+  bool isSMEMOffset() const;
   bool isSMRDLiteralOffset() const;
   bool isDPP8() const;
   bool isDPPCtrl() const;
   bool isBLGP() const;
   bool isCBSZ() const;
   bool isABID() const;
   bool isGPRIdxMode() const;
   bool isS16Imm() const;
private:
   bool validateHwreg(const OperandInfoTy &HwReg,
                      const int64_t Offset,
                      const int64_t Width,
                      const SMLoc Loc);
 
   void errorExpTgt();
   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
   SMLoc getFlatOffsetLoc(const OperandVector &Operands) const;
+  SMLoc getSMEMOffsetLoc(const OperandVector &Operands) const;
 
   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc, const OperandVector &Operands);
   bool validateFlatOffset(const MCInst &Inst, const OperandVector &Operands);
+  bool validateSMEMOffset(const MCInst &Inst, const OperandVector &Operands);
   bool validateSOPLiteral(const MCInst &Inst) const;
   bool validateConstantBusLimitations(const MCInst &Inst);
   bool validateEarlyClobberLimitations(const MCInst &Inst);
   bool validateIntClampSupported(const MCInst &Inst);
   bool validateMIMGAtomicDMask(const MCInst &Inst);
   bool validateMIMGGatherDMask(const MCInst &Inst);
   bool validateMovrels(const MCInst &Inst);
   bool validateMIMGDataSize(const MCInst &Inst);
public:
   void cvtMubufLds(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false, true); }
   void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
 
   AMDGPUOperand::Ptr defaultDLC() const;
   AMDGPUOperand::Ptr defaultGLC() const;
   AMDGPUOperand::Ptr defaultSLC() const;
 
   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
-  AMDGPUOperand::Ptr defaultSMRDOffset20() const;
+  AMDGPUOperand::Ptr defaultSMEMOffset() const;
   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
   AMDGPUOperand::Ptr defaultFlatOffset() const;
 
   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
 
   void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
                OptionalImmIndexMap &OptionalIdx);
   void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
if (!isUIntN(OffsetSize - 1, Op.getImm())) {
                        "expected an 11-bit unsigned offset");
       return false;
     }
   }
 
   return true;
 }
 
+SMLoc AMDGPUAsmParser::getSMEMOffsetLoc(const OperandVector &Operands) const {
+  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
+    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
+    if (Op.isSMEMOffset())
+      return Op.getStartLoc();
+  }
+  return getLoc();
+}
+
+bool AMDGPUAsmParser::validateSMEMOffset(const MCInst &Inst,
+                                         const OperandVector &Operands) {
+  if (isCI() || isSI())
+    return true;
+
+  uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
+  if ((TSFlags & SIInstrFlags::SMRD) == 0)
+    return true;
+
+  auto Opcode = Inst.getOpcode();
+  auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
+  if (OpNum == -1)
+    return true;
+
+  const auto &Op = Inst.getOperand(OpNum);
+  if (!Op.isImm())
+    return true;
+
+  uint64_t Offset = Op.getImm();
+  bool IsBuffer = AMDGPU::getSMEMIsBuffer(Opcode);
+  if (AMDGPU::isLegalSMRDEncodedUnsignedOffset(getSTI(), Offset) ||
+      AMDGPU::isLegalSMRDEncodedSignedOffset(getSTI(), Offset, IsBuffer))
+    return true;
+
+  Error(getSMEMOffsetLoc(Operands),
+        (isVI() || IsBuffer) ? "expected a 20-bit unsigned offset" :
+                               "expected a 21-bit signed offset");
+
+  return false;
+}
+
 bool AMDGPUAsmParser::validateSOPLiteral(const MCInst &Inst) const {
   unsigned Opcode = Inst.getOpcode();
   const MCInstrDesc &Desc = MII.get(Opcode);
   if (!(Desc.TSFlags & (SIInstrFlags::SOP2 | SIInstrFlags::SOPC)))
     return true;
 
   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
   const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
 
   const int OpIndices[] = { Src0Idx, Src1Idx };
bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
   }
   if (!validateMovrels(Inst)) {
     Error(IDLoc, "source operand must be a VGPR");
     return false;
   }
   if (!validateFlatOffset(Inst, Operands)) {
     return false;
   }
+  if (!validateSMEMOffset(Inst, Operands)) {
+    return false;
+  }
 
   return true;
 }
 
 static std::string AMDGPUMnemonicSpellCheck(StringRef S,
                                             const FeatureBitset &FBS,
                                             unsigned VariantID = 0);
 

 //===----------------------------------------------------------------------===//
 // smrd
 //===----------------------------------------------------------------------===//
 
 bool AMDGPUOperand::isSMRDOffset8() const {
   return isImm() && isUInt<8>(getImm());
 }
 
-bool AMDGPUOperand::isSMRDOffset20() const {
-  return isImm() && isUInt<20>(getImm());
+bool AMDGPUOperand::isSMEMOffset() const {
+  return isImm(); // Offset range is checked later by validator.
 }
 
 bool AMDGPUOperand::isSMRDLiteralOffset() const {
   // 32-bit literals are only supported on CI and we only want to use them
   // when the offset is > 8-bits.
   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
 }
 
 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
 }
 
-AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
+AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMEMOffset() const {
   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
 }
 
 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
 }
 
 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultFlatOffset() const {
unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
   case MCK_VReg32OrOff:
     return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
   case MCK_InterpSlot:
     return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
   case MCK_Attr:
     return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
   case MCK_AttrChan:
     return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
+  case MCK_ImmSMEMOffset:
+    return Operand.isSMEMOffset() ? Match_Success : Match_InvalidOperand;
   case MCK_SReg_64:
   case MCK_SReg_64_XEXEC:
     // Null is defined as a 32-bit register but
     // it should also be enabled with 64-bit operands.
     // The following code enables it for SReg_64 operands
     // used as source and destination. Remaining source
     // operands are handled in isInlinableImm.
     return Operand.isNull() ? Match_Success : Match_InvalidOperand;

llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp

static DecodeStatus decodeSoppBrTarget(MCInst &Inst, unsigned Imm,
   APInt SignedOffset(18, Imm * 4, true);
   int64_t Offset = (SignedOffset.sext(64) + 4 + Addr).getSExtValue();
 
   if (DAsm->tryAddingSymbolicOperand(Inst, Offset, Addr, true, 2, 2))
     return MCDisassembler::Success;
   return addOperand(Inst, MCOperand::createImm(Imm));
 }
 
+static DecodeStatus decodeSMEMOffset(MCInst &Inst, unsigned Imm,
+                                     uint64_t Addr, const void *Decoder) {
+  auto DAsm = static_cast<const AMDGPUDisassembler*>(Decoder);
+  int64_t Offset;
+  if (DAsm->isVI()) {         // VI supports 20-bit unsigned offsets.
+    Offset = Imm & 0xFFFFF;
+  } else {                    // GFX9+ supports 21-bit signed offsets.
+    Offset = SignExtend64<21>(Imm);
+  }
+  return addOperand(Inst, MCOperand::createImm(Offset));
+}
+
 static DecodeStatus decodeBoolReg(MCInst &Inst, unsigned Val,
                                   uint64_t Addr, const void *Decoder) {
   auto DAsm = static_cast<const AMDGPUDisassembler*>(Decoder);
   return addOperand(Inst, DAsm->decodeBoolReg(Val));
 }
 
 #define DECODE_OPERAND(StaticDecoderName, DecoderName) \
 static DecodeStatus StaticDecoderName(MCInst &Inst, \

llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h

void printFlatOffset(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                        raw_ostream &O);
 
   void printOffset0(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                     raw_ostream &O);
   void printOffset1(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                     raw_ostream &O);
   void printSMRDOffset8(const MCInst *MI, unsigned OpNo,
                        const MCSubtargetInfo &STI, raw_ostream &O);
-  void printSMRDOffset20(const MCInst *MI, unsigned OpNo,
+  void printSMEMOffset(const MCInst *MI, unsigned OpNo,
                        const MCSubtargetInfo &STI, raw_ostream &O);
   void printSMRDLiteralOffset(const MCInst *MI, unsigned OpNo,
                               const MCSubtargetInfo &STI, raw_ostream &O);
   void printGDS(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                 raw_ostream &O);
   void printDLC(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                 raw_ostream &O);
   void printGLC(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,

llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp

 }
 
 void AMDGPUInstPrinter::printSMRDOffset8(const MCInst *MI, unsigned OpNo,
                                         const MCSubtargetInfo &STI,
                                         raw_ostream &O) {
   printU32ImmOperand(MI, OpNo, STI, O);
 }
 
-void AMDGPUInstPrinter::printSMRDOffset20(const MCInst *MI, unsigned OpNo,
+void AMDGPUInstPrinter::printSMEMOffset(const MCInst *MI, unsigned OpNo,
                                         const MCSubtargetInfo &STI,
                                         raw_ostream &O) {
-  printU32ImmOperand(MI, OpNo, STI, O);
+  O << formatHex(MI->getOperand(OpNo).getImm());
 }
 
 void AMDGPUInstPrinter::printSMRDLiteralOffset(const MCInst *MI, unsigned OpNo,
                                                const MCSubtargetInfo &STI,
                                                raw_ostream &O) {
   printU32ImmOperand(MI, OpNo, STI, O);
 }
 

llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.h

public:
   }
 
   virtual unsigned getSOPPBrEncoding(const MCInst &MI, unsigned OpNo,
                                      SmallVectorImpl<MCFixup> &Fixups,
                                      const MCSubtargetInfo &STI) const {
     return 0;
   }
 
+  virtual unsigned getSMEMOffsetEncoding(const MCInst &MI, unsigned OpNo,
+                                         SmallVectorImpl<MCFixup> &Fixups,
+                                         const MCSubtargetInfo &STI) const {
+    return 0;
+  }
+
   virtual unsigned getSDWASrcEncoding(const MCInst &MI, unsigned OpNo,
                                       SmallVectorImpl<MCFixup> &Fixups,
                                       const MCSubtargetInfo &STI) const {
     return 0;
   }
 
   virtual unsigned getSDWAVopcDstEncoding(const MCInst &MI, unsigned OpNo,
                                           SmallVectorImpl<MCFixup> &Fixups,

llvm/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp

uint64_t getMachineOpValue(const MCInst &MI, const MCOperand &MO,
                              const MCSubtargetInfo &STI) const override;
 
   /// Use a fixup to encode the simm16 field for SOPP branch
   ///        instructions.
   unsigned getSOPPBrEncoding(const MCInst &MI, unsigned OpNo,
                              SmallVectorImpl<MCFixup> &Fixups,
                              const MCSubtargetInfo &STI) const override;
 
+  unsigned getSMEMOffsetEncoding(const MCInst &MI, unsigned OpNo,
+                                 SmallVectorImpl<MCFixup> &Fixups,
+                                 const MCSubtargetInfo &STI) const override;
+
   unsigned getSDWASrcEncoding(const MCInst &MI, unsigned OpNo,
                               SmallVectorImpl<MCFixup> &Fixups,
                               const MCSubtargetInfo &STI) const override;
 
   unsigned getSDWAVopcDstEncoding(const MCInst &MI, unsigned OpNo,
                                   SmallVectorImpl<MCFixup> &Fixups,
                                   const MCSubtargetInfo &STI) const override;
 
if (MO.isExpr()) {
     MCFixupKind Kind = (MCFixupKind)AMDGPU::fixup_si_sopp_br;
     Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
     return 0;
   }
 
   return getMachineOpValue(MI, MO, Fixups, STI);
 }
 
+unsigned SIMCCodeEmitter::getSMEMOffsetEncoding(const MCInst &MI, unsigned OpNo,
+                                                SmallVectorImpl<MCFixup> &Fixups,
+                                                const MCSubtargetInfo &STI) const {
+  const MCOperand &MO = MI.getOperand(OpNo);
+  assert(MO.isImm());
+
+  auto Offset = MO.getImm();
+  if (AMDGPU::isVI(STI)) {
+    Offset &= 0xFFFFF;       // VI only supports 20-bit unsigned offsets.
+  }
+  return Offset;
+}
+
 unsigned
 SIMCCodeEmitter::getSDWASrcEncoding(const MCInst &MI, unsigned OpNo,
                                     SmallVectorImpl<MCFixup> &Fixups,
                                     const MCSubtargetInfo &STI) const {
   using namespace AMDGPU::SDWA;
 
   uint64_t RegEnc = 0;
 

llvm/lib/Target/AMDGPU/SMInstructions.td

 //===---- SMInstructions.td - Scalar Memory Instruction Definitions -------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 def smrd_offset_8 : NamedOperandU32<"SMRDOffset8",
                                   NamedMatchClass<"SMRDOffset8">> {
   let OperandType = "OPERAND_IMMEDIATE";
 }
 
-def smrd_offset_20 : NamedOperandU32<"SMRDOffset20",
-                                  NamedMatchClass<"SMRDOffset20">> {
+def smem_offset : NamedOperandU32<"SMEMOffset",
+                                  NamedMatchClass<"SMEMOffset">> {
   let OperandType = "OPERAND_IMMEDIATE";
+  let EncoderMethod = "getSMEMOffsetEncoding";
+  let DecoderMethod = "decodeSMEMOffset";
 }
 
 //===----------------------------------------------------------------------===//
 // Scalar Memory classes
 //===----------------------------------------------------------------------===//
 
 class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> :
   InstSI <outs, ins, "", pattern>,
class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> :
   string AsmOperands = asmOps;
 
   bits<1> has_sbase = 1;
   bits<1> has_sdst = 1;
   bit has_glc = 0;
   bit has_dlc = 0;
   bits<1> has_offset = 1;
   bits<1> offset_is_imm = 0;
+  bit is_buffer = 0;
 }
 
 class SM_Real <SM_Pseudo ps>
   : InstSI<ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 
   let isPseudo = 0;
   let isCodeGenOnly = 0;
 
+  Instruction Opcode = !cast<Instruction>(NAME);
+
   // copy relevant pseudo op flags
   let SubtargetPredicate = ps.SubtargetPredicate;
   let AsmMatchConverter  = ps.AsmMatchConverter;
+  let UseNamedOperandTable = ps.UseNamedOperandTable;
+  let SMRD = ps.SMRD;
+
+  bit is_buffer = ps.is_buffer;
 
   // encoding
   bits<7>  sbase;
   bits<7>  sdst;
   bits<32> offset;
   bits<1> imm = !if(ps.has_offset, ps.offset_is_imm, 0);
 }
 
def _SGPR  : SM_Store_Pseudo <opName,
     " $sdata, $sbase, $offset$glc$dlc", []> {
     let BaseClass = baseClass;
     let SrcClass = srcClass;
     let PseudoInstr = opName # "_SGPR";
   }
 }
 
 multiclass SM_Pseudo_Discards<string opName> {
-  def _IMM  : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, smrd_offset_20:$offset), 1>;
+  def _IMM  : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, smem_offset:$offset), 1>;
   def _SGPR : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, SReg_32:$offset), 0>;
 }
 
 class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pseudo<
   opName, (outs SReg_64_XEXEC:$sdst), (ins),
   " $sdst", [(set i64:$sdst, (node))]> {
   let hasSideEffects = 1;
 
class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_Pseudo<
   let hasSideEffects = 1;
   let mayStore = 1;
   let has_sdst = 0;
   let has_sbase = 0;
   let has_offset = 0;
 }
 
 multiclass SM_Pseudo_Probe<string opName, RegisterClass baseClass> {
-  def _IMM  : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, smrd_offset_20:$offset), 1>;
+  def _IMM  : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, smem_offset:$offset), 1>;
   def _SGPR : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, SReg_32:$offset), 0>;
 }
 
 class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo<
   opName, (outs SReg_32_XM0_XEXEC:$sdst), (ins),
   " $sdst", [(set i32:$sdst, (node))]> {
   let hasSideEffects = 1;
   let mayStore = 0;

 class SM_Pseudo_Atomic<string opName,
                        RegisterClass baseClass,
                        RegisterClass dataClass,
                        bit isImm,
                        bit isRet> :
   SM_Atomic_Pseudo<opName,
                    !if(isRet, (outs dataClass:$sdst), (outs)),
                    !if(isImm,
-                       (ins dataClass:$sdata, baseClass:$sbase, smrd_offset_20:$offset, DLC:$dlc),
+                       (ins dataClass:$sdata, baseClass:$sbase, smem_offset:$offset, DLC:$dlc),
                        (ins dataClass:$sdata, baseClass:$sbase, SReg_32:$offset, DLC:$dlc)),
                    !if(isRet, " $sdst", " $sdata") # ", $sbase, $offset" # !if(isRet, " glc", "") # "$dlc",
                    isRet> {
   let offset_is_imm = isImm;
   let PseudoInstr = opName # !if(isImm,
                                  !if(isRet, "_IMM_RTN", "_IMM"),
                                  !if(isRet, "_SGPR_RTN", "_SGPR"));
 

 // XXX - SMEM instructions do not allow exec for data operand, but
 // does sdst for SMRD on SI/CI?
 defm S_LOAD_DWORD    : SM_Pseudo_Loads <"s_load_dword", SReg_64, SReg_32_XM0_XEXEC>;
 defm S_LOAD_DWORDX2  : SM_Pseudo_Loads <"s_load_dwordx2", SReg_64, SReg_64_XEXEC>;
 defm S_LOAD_DWORDX4  : SM_Pseudo_Loads <"s_load_dwordx4", SReg_64, SReg_128>;
 defm S_LOAD_DWORDX8  : SM_Pseudo_Loads <"s_load_dwordx8", SReg_64, SReg_256>;
 defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <"s_load_dwordx16", SReg_64, SReg_512>;
 
+let is_buffer = 1 in {
 defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads <
   "s_buffer_load_dword", SReg_128, SReg_32_XM0_XEXEC
 >;
 
 // FIXME: exec_lo/exec_hi appear to be allowed for SMRD loads on
 // SI/CI, bit disallowed for SMEM on VI.
 defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads <
   "s_buffer_load_dwordx2", SReg_128, SReg_64_XEXEC
 >;
 
 defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads <
   "s_buffer_load_dwordx4", SReg_128, SReg_128
 >;
 
 defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads <
   "s_buffer_load_dwordx8", SReg_128, SReg_256
 >;
 
 defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads <
   "s_buffer_load_dwordx16", SReg_128, SReg_512
 >;
+}
 
 let SubtargetPredicate = HasScalarStores in {
 defm S_STORE_DWORD : SM_Pseudo_Stores <"s_store_dword", SReg_64, SReg_32_XM0_XEXEC>;
 defm S_STORE_DWORDX2 : SM_Pseudo_Stores <"s_store_dwordx2", SReg_64, SReg_64_XEXEC>;
 defm S_STORE_DWORDX4 : SM_Pseudo_Stores <"s_store_dwordx4", SReg_64, SReg_128>;
 
+let is_buffer = 1 in {
 defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores <
   "s_buffer_store_dword", SReg_128, SReg_32_XM0_XEXEC
 >;
 
 defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores <
   "s_buffer_store_dwordx2", SReg_128, SReg_64_XEXEC
 >;
 
 defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores <
   "s_buffer_store_dwordx4", SReg_128, SReg_128
 >;
+}
 } // End SubtargetPredicate = HasScalarStores
 
 def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>;
 def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>;
 
 let SubtargetPredicate = isGFX7GFX8GFX9 in {
 def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>;
 } // let SubtargetPredicate = isGFX7GFX8GFX9
 
 let SubtargetPredicate = isGFX8Plus in {
 let OtherPredicates = [HasScalarStores] in {
 def S_DCACHE_WB     : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>;
 def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>;
 } // End OtherPredicates = [HasScalarStores]
 def S_MEMREALTIME   : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>;
 
 defm S_ATC_PROBE        : SM_Pseudo_Probe <"s_atc_probe", SReg_64>;
+let is_buffer = 1 in {
 defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <"s_atc_probe_buffer", SReg_128>;
+}
 } // SubtargetPredicate = isGFX8Plus
 
 let SubtargetPredicate = isGFX10Plus in {
 def S_GL1_INV : SM_Inval_Pseudo<"s_gl1_inv">;
 def S_GET_WAVEID_IN_WORKGROUP : SM_WaveId_Pseudo <"s_get_waveid_in_workgroup", int_amdgcn_s_get_waveid_in_workgroup>;
 } // End SubtargetPredicate = isGFX10Plus
 
 let SubtargetPredicate = HasScalarFlatScratchInsts, Uses = [FLAT_SCR] in {
 defm S_SCRATCH_LOAD_DWORD    : SM_Pseudo_Loads <"s_scratch_load_dword",   SReg_64, SReg_32_XM0_XEXEC>;
 defm S_SCRATCH_LOAD_DWORDX2  : SM_Pseudo_Loads <"s_scratch_load_dwordx2", SReg_64, SReg_64_XEXEC>;
 defm S_SCRATCH_LOAD_DWORDX4  : SM_Pseudo_Loads <"s_scratch_load_dwordx4", SReg_64, SReg_128>;
 
 defm S_SCRATCH_STORE_DWORD   : SM_Pseudo_Stores <"s_scratch_store_dword",   SReg_64, SReg_32_XM0_XEXEC>;
 defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <"s_scratch_store_dwordx2", SReg_64, SReg_64_XEXEC>;
 defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <"s_scratch_store_dwordx4", SReg_64, SReg_128>;
 } // SubtargetPredicate = HasScalarFlatScratchInsts
 
 let SubtargetPredicate = HasScalarAtomics in {
 
+let is_buffer = 1 in {
 defm S_BUFFER_ATOMIC_SWAP         : SM_Pseudo_Atomics <"s_buffer_atomic_swap", SReg_128, SReg_32_XM0_XEXEC>;
 defm S_BUFFER_ATOMIC_CMPSWAP      : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap", SReg_128, SReg_64_XEXEC>;
 defm S_BUFFER_ATOMIC_ADD          : SM_Pseudo_Atomics <"s_buffer_atomic_add", SReg_128, SReg_32_XM0_XEXEC>;
 defm S_BUFFER_ATOMIC_SUB          : SM_Pseudo_Atomics <"s_buffer_atomic_sub", SReg_128, SReg_32_XM0_XEXEC>;
 defm S_BUFFER_ATOMIC_SMIN         : SM_Pseudo_Atomics <"s_buffer_atomic_smin", SReg_128, SReg_32_XM0_XEXEC>;
 defm S_BUFFER_ATOMIC_UMIN         : SM_Pseudo_Atomics <"s_buffer_atomic_umin", SReg_128, SReg_32_XM0_XEXEC>;
 defm S_BUFFER_ATOMIC_SMAX         : SM_Pseudo_Atomics <"s_buffer_atomic_smax", SReg_128, SReg_32_XM0_XEXEC>;
 defm S_BUFFER_ATOMIC_UMAX         : SM_Pseudo_Atomics <"s_buffer_atomic_umax", SReg_128, SReg_32_XM0_XEXEC>;

 defm S_BUFFER_ATOMIC_UMIN_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_umin_x2", SReg_128, SReg_64_XEXEC>;
 defm S_BUFFER_ATOMIC_SMAX_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_smax_x2", SReg_128, SReg_64_XEXEC>;
 defm S_BUFFER_ATOMIC_UMAX_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_umax_x2", SReg_128, SReg_64_XEXEC>;
 defm S_BUFFER_ATOMIC_AND_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_and_x2", SReg_128, SReg_64_XEXEC>;
 defm S_BUFFER_ATOMIC_OR_X2        : SM_Pseudo_Atomics <"s_buffer_atomic_or_x2", SReg_128, SReg_64_XEXEC>;
 defm S_BUFFER_ATOMIC_XOR_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_xor_x2", SReg_128, SReg_64_XEXEC>;
 defm S_BUFFER_ATOMIC_INC_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_inc_x2", SReg_128, SReg_64_XEXEC>;
 defm S_BUFFER_ATOMIC_DEC_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_dec_x2", SReg_128, SReg_64_XEXEC>;
+}
 
 defm S_ATOMIC_SWAP                : SM_Pseudo_Atomics <"s_atomic_swap", SReg_64, SReg_32_XM0_XEXEC>;
 defm S_ATOMIC_CMPSWAP             : SM_Pseudo_Atomics <"s_atomic_cmpswap", SReg_64, SReg_64_XEXEC>;
 defm S_ATOMIC_ADD                 : SM_Pseudo_Atomics <"s_atomic_add", SReg_64, SReg_32_XM0_XEXEC>;
 defm S_ATOMIC_SUB                 : SM_Pseudo_Atomics <"s_atomic_sub", SReg_64, SReg_32_XM0_XEXEC>;
 defm S_ATOMIC_SMIN                : SM_Pseudo_Atomics <"s_atomic_smin", SReg_64, SReg_32_XM0_XEXEC>;
 defm S_ATOMIC_UMIN                : SM_Pseudo_Atomics <"s_atomic_umin", SReg_64, SReg_32_XM0_XEXEC>;
 defm S_ATOMIC_SMAX                : SM_Pseudo_Atomics <"s_atomic_smax", SReg_64, SReg_32_XM0_XEXEC>;
class SMEM_Real_vi <bits<8> op, SM_Pseudo ps>
 
   let Inst{5-0}   = !if(ps.has_sbase, sbase{6-1}, ?);
   let Inst{12-6}  = !if(ps.has_sdst, sdst{6-0}, ?);
 
   let Inst{16} = !if(ps.has_glc, glc, ?);
   let Inst{17} = imm;
   let Inst{25-18} = op;
   let Inst{31-26} = 0x30; //encoding
-  let Inst{51-32} = !if(ps.has_offset, offset{19-0}, ?);
+  
+  // VI supports 20-bit unsigned offsets while GFX9+ supports 21-bit signed.
+  // Offset value is corrected accordingly when offset is encoded/decoded.
+  let Inst{52-32} = !if(ps.has_offset, offset{20-0}, ?);
 }
 
 multiclass SM_Real_Loads_vi<bits<8> op, string ps,
                             SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM),
                             SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> {
   def _IMM_vi : SMEM_Real_vi <op, immPs> {
-    let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_20:$offset, GLC:$glc, DLC:$dlc);
+    let InOperandList = (ins immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc);
   }
   def _SGPR_vi : SMEM_Real_vi <op, sgprPs> {
     let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc);
   }
 }
 
 class SMEM_Real_Store_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> {
   // encoding
   bits<7> sdata;
 
   let sdst = ?;
   let Inst{12-6}  = !if(ps.has_sdst, sdata{6-0}, ?);
 }
 
 multiclass SM_Real_Stores_vi<bits<8> op, string ps,
                             SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM),
                             SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> {
   // FIXME: The operand name $offset is inconsistent with $soff used
   // in the pseudo
   def _IMM_vi : SMEM_Real_Store_vi <op, immPs> {
-    let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smrd_offset_20:$offset, GLC:$glc, DLC:$dlc);
+    let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc);
   }
 
   def _SGPR_vi : SMEM_Real_Store_vi <op, sgprPs> {
     let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc);
   }
 }
 
 multiclass SM_Real_Probe_vi<bits<8> op, string ps> {
class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> :
   SM_Real<ps>,
   Enc64 {
 
   let AssemblerPredicate = isGFX7Only;
   let DecoderNamespace = "GFX7";
   let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, GLC:$glc, DLC:$dlc);
 
   let LGKM_CNT = ps.LGKM_CNT;
-  let SMRD = ps.SMRD;
   let mayLoad = ps.mayLoad;
   let mayStore = ps.mayStore;
   let hasSideEffects = ps.hasSideEffects;
   let SchedRW = ps.SchedRW;
-  let UseNamedOperandTable = ps.UseNamedOperandTable;
 
   let Inst{7-0}   = 0xff;
   let Inst{8}     = 0;
   let Inst{14-9}  = sbase{6-1};
   let Inst{21-15} = sdst{6-0};
   let Inst{26-22} = op;
   let Inst{31-27} = 0x18; //encoding
   let Inst{63-32} = offset{31-0};
class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps> :
   let DecoderNamespace = "GFX10";
 
   let Inst{5-0}   = !if(ps.has_sbase, sbase{6-1}, ?);
   let Inst{12-6}  = !if(ps.has_sdst, sdst{6-0}, ?);
   let Inst{14}    = !if(ps.has_dlc, dlc, ?);
   let Inst{16}    = !if(ps.has_glc, glc, ?);
   let Inst{25-18} = op;
   let Inst{31-26} = 0x3d;
-  let Inst{51-32} = !if(ps.offset_is_imm, !if(ps.has_offset, offset{19-0}, ?), ?);
+  let Inst{52-32} = !if(ps.offset_is_imm, !if(ps.has_offset, offset{20-0}, ?), ?);
   let Inst{63-57} = !if(ps.offset_is_imm, !cast<int>(SGPR_NULL.HWEncoding),
                                           !if(ps.has_offset, offset{6-0}, ?));
 }
 
 multiclass SM_Real_Loads_gfx10<bits<8> op, string ps,
                                SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM),
                                SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> {
   def _IMM_gfx10 : SMEM_Real_gfx10<op, immPs> {
-    let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_20:$offset, GLC:$glc, DLC:$dlc);
+    let InOperandList = (ins immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc);
   }
   def _SGPR_gfx10 : SMEM_Real_gfx10<op, sgprPs> {
     let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc);
   }
 }
 
 class SMEM_Real_Store_gfx10<bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx10<op, ps> {
   bits<7> sdata;
 
   let sdst = ?;
   let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?);
 }
 
 multiclass SM_Real_Stores_gfx10<bits<8> op, string ps,
                                 SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM),
                                 SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> {
   // FIXME: The operand name $offset is inconsistent with $soff used
   // in the pseudo
   def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, immPs> {
-    let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smrd_offset_20:$offset, GLC:$glc, DLC:$dlc);
+    let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc);
   }
 
   def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, sgprPs> {
     let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc);
   }
 }
 
 defm S_LOAD_DWORD            : SM_Real_Loads_gfx10<0x000, "S_LOAD_DWORD">;
multiclass SM_Real_Discard_gfx10<bits<8> op, string ps> {
   def _IMM_gfx10  : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>;
   def _SGPR_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>;
 }
 
 defm S_DCACHE_DISCARD    : SM_Real_Discard_gfx10 <0x28, "S_DCACHE_DISCARD">;
 defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_gfx10 <0x29, "S_DCACHE_DISCARD_X2">;
 
 } // End SubtargetPredicate = HasScalarAtomics
+
+def SMInfoTable : GenericTable {
+  let FilterClass = "SM_Real";
+  let CppTypeName = "SMInfo";
+  let Fields = ["Opcode", "is_buffer"];
+
+  let PrimaryKey = ["Opcode"];
+  let PrimaryKeyName = "getSMEMOpcodeHelper";
+}

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h

 
 LLVM_READONLY
 bool getMUBUFHasSrsrc(unsigned Opc);
 
 LLVM_READONLY
 bool getMUBUFHasSoffset(unsigned Opc);
 
 LLVM_READONLY
+bool getSMEMIsBuffer(unsigned Opc);
+
+LLVM_READONLY
 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,
                                                   uint8_t NumComponents,
                                                   uint8_t NumFormat,
                                                   const MCSubtargetInfo &STI);
 LLVM_READONLY
 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,
                                                   const MCSubtargetInfo &STI);
 

 LLVM_READNONE
 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi);
 
 LLVM_READNONE
 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi);
 
 bool isArgPassedInSGPR(const Argument *Arg);
 
+LLVM_READONLY
+bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
+                                      int64_t EncodedOffset);
+
+LLVM_READONLY
+bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
+                                    int64_t EncodedOffset,
+                                    bool IsBuffer);
+
 /// Convert \p ByteOffset to dwords if the subtarget uses dword SMRD immediate
 /// offsets.
 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, uint64_t ByteOffset);
 
 /// \returns The encoding that will be used for \p ByteOffset in the
 /// SMRD offset field, or None if it won't fit. On GFX9 and GFX10
 /// S_LOAD instructions have a signed offset, on other subtargets it is
 /// unsigned. S_BUFFER has an unsigned offset for all subtargets.

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

struct MTBUFInfo {
   uint16_t Opcode;
   uint16_t BaseOpcode;
   uint8_t elements;
   bool has_vaddr;
   bool has_srsrc;
   bool has_soffset;
 };
 
+struct SMInfo {
+  uint16_t Opcode;
+  bool IsBuffer;
+};
+
 #define GET_MTBUFInfoTable_DECL
 #define GET_MTBUFInfoTable_IMPL
 #define GET_MUBUFInfoTable_DECL
 #define GET_MUBUFInfoTable_IMPL
+#define GET_SMInfoTable_DECL
+#define GET_SMInfoTable_IMPL
 #include "AMDGPUGenSearchableTables.inc"
 
 int getMTBUFBaseOpcode(unsigned Opc) {
   const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc);
   return Info ? Info->BaseOpcode : -1;
 }
 
 int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {
bool getMUBUFHasSrsrc(unsigned Opc) {
   return Info ? Info->has_srsrc : false;
 }
 
 bool getMUBUFHasSoffset(unsigned Opc) {
   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
   return Info ? Info->has_soffset : false;
 }
 
+bool getSMEMIsBuffer(unsigned Opc) {
+  const SMInfo *Info = getSMEMOpcodeHelper(Opc);
+  return Info ? Info->IsBuffer : false;
+}
+
 // Wrapper for Tablegen'd function.  enum Subtarget is not defined in any
 // header files, so we need to wrap it in a function that takes unsigned
 // instead.
 int getMCOpcode(uint16_t Opcode, unsigned Gen) {
   return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen));
 }
 
 namespace IsaInfo {

 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
   return isGCN3Encoding(ST) || isGFX10(ST);
 }
 
 static bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) {
   return isGFX9(ST) || isGFX10(ST);
 }
 
-static bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
-                                             int64_t EncodedOffset) {
+bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
   return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset)
                                : isUInt<8>(EncodedOffset);
 }
 
+bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
+                                    int64_t EncodedOffset,
+                                    bool IsBuffer) {
+  return !IsBuffer &&
+         hasSMRDSignedImmOffset(ST) &&
+         isInt<21>(EncodedOffset);
+}
+
 static bool isDwordAligned(uint64_t ByteOffset) {
   return (ByteOffset & 3) == 0;
 }
 
 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST,
                                 uint64_t ByteOffset) {
   if (hasSMEMByteOffset(ST))
     return ByteOffset;
 
   assert(isDwordAligned(ByteOffset));
   return ByteOffset >> 2;
 }
 
 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
                                        int64_t ByteOffset, bool IsBuffer) {
   // The signed version is always a byte offset.
   if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {
     assert(hasSMEMByteOffset(ST));
     return isInt<20>(ByteOffset) ? Optional<int64_t>(ByteOffset) : None;
   }
 
   if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
     return None;
 
   int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
   return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)
              ? Optional<int64_t>(EncodedOffset)

llvm/test/CodeGen/AMDGPU/GlobalISel/smrd.ll

entry:
   store i32 %1, i32 addrspace(1)* undef
   ret void
 }
 
 ; GFX9_10 can use a signed immediate byte offset
 ; GCN-LABEL: {{^}}smrd6:
 ; SICIVI: s_add_u32 s{{[0-9]}}, s{{[0-9]}}, -4
 ; SICIVI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x0
-; GFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xfffffffc
+; GFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], -0x4
 define amdgpu_kernel void @smrd6(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 {
 entry:
   %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 -1
   %tmp1 = load i32, i32 addrspace(4)* %tmp
   store i32 %tmp1, i32 addrspace(1)* %out
   ret void
 }
 

llvm/test/CodeGen/AMDGPU/smrd.ll

entry:
   store i32 %tmp1, i32 addrspace(1)* %out
   ret void
 }
 
 ; GFX9_10 can use a signed immediate byte offset
 ; GCN-LABEL: {{^}}smrd6:
 ; SICIVI: s_add_u32 s{{[0-9]}}, s{{[0-9]}}, -4
 ; SICIVI: s_load_dword s{{[0-9]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
-; GFX9_10: s_load_dword s{{[0-9]}}, s{{\[[0-9]+:[0-9]+\]}}, 0xfffffffc
+; GFX9_10: s_load_dword s{{[0-9]}}, s{{\[[0-9]+:[0-9]+\]}}, -0x4
 define amdgpu_kernel void @smrd6(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 {
 entry:
   %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 -1
   %tmp1 = load i32, i32 addrspace(4)* %tmp
   store i32 %tmp1, i32 addrspace(1)* %out
   ret void
 }
 

llvm/test/MC/AMDGPU/smem.s

 // GFX9:     s_buffer_atomic_xor s5, s[4:7], s0 ; encoding: [0x42,0x01,0x28,0xc1,0x00,0x00,0x00,0x00]
 // GFX1012:  s_buffer_atomic_xor s5, s[4:7], s0 ; encoding: [0x42,0x01,0x28,0xf5,0x00,0x00,0x00,0x00]
 // NOSICIVI: error:
 
 s_buffer_atomic_xor_x2 s[10:11], s[4:7], s0 glc
 // GFX9:     s_buffer_atomic_xor_x2 s[10:11], s[4:7], s0 glc ; encoding: [0x82,0x02,0xa9,0xc1,0x00,0x00,0x00,0x00]
 // GFX1012:  s_buffer_atomic_xor_x2 s[10:11], s[4:7], s0 glc ; encoding: [0x82,0x02,0xa9,0xf5,0x00,0x00,0x00,0x00]
 // NOSICIVI: error:
+
+//===----------------------------------------------------------------------===//
+// Unsigned 20-bit offsets (VI+)
+//===----------------------------------------------------------------------===//
+
+s_atc_probe 0x7, s[4:5], 0xFFFFF
+// NOSICI: error: instruction not supported on this GPU
+// GFX89: s_atc_probe 7, s[4:5], 0xfffff ; encoding: [0xc2,0x01,0x9a,0xc0,0xff,0xff,0x0f,0x00]
+// GFX10: s_atc_probe 7, s[4:5], 0xfffff ; encoding: [0xc2,0x01,0x98,0xf4,0xff,0xff,0x0f,0xfa]
+
+s_atc_probe_buffer 0x1, s[8:11], 0xFFFFF
+// NOSICI: error: instruction not supported on this GPU
+// GFX89: s_atc_probe_buffer 1, s[8:11], 0xfffff ; encoding: [0x44,0x00,0x9e,0xc0,0xff,0xff,0x0f,0x00]
+// GFX10: s_atc_probe_buffer 1, s[8:11], 0xfffff ; encoding: [0x44,0x00,0x9c,0xf4,0xff,0xff,0x0f,0xfa]
+
+s_store_dword s1, s[2:3], 0xFFFFF
+// NOSICI: error: instruction not supported on this GPU
+// GFX89: s_store_dword s1, s[2:3], 0xfffff ; encoding: [0x41,0x00,0x42,0xc0,0xff,0xff,0x0f,0x00]
+// GFX10: s_store_dword s1, s[2:3], 0xfffff ; encoding: [0x41,0x00,0x40,0xf4,0xff,0xff,0x0f,0xfa]
+
+s_buffer_store_dword s10, s[92:95], 0xFFFFF
+// NOSICI: error: instruction not supported on this GPU
+// GFX89: s_buffer_store_dword s10, s[92:95], 0xfffff ; encoding: [0xae,0x02,0x62,0xc0,0xff,0xff,0x0f,0x00]
+// GFX10: s_buffer_store_dword s10, s[92:95], 0xfffff ; encoding: [0xae,0x02,0x60,0xf4,0xff,0xff,0x0f,0xfa]
+
+s_atomic_swap s5, s[2:3], 0xFFFFF
+// NOSICIVI: error: instruction not supported on this GPU
+// GFX10: s_atomic_swap s5, s[2:3], 0xfffff ; encoding: [0x41,0x01,0x00,0xf6,0xff,0xff,0x0f,0xfa]
+// GFX9: s_atomic_swap s5, s[2:3], 0xfffff ; encoding: [0x41,0x01,0x02,0xc2,0xff,0xff,0x0f,0x00]
+
+s_buffer_atomic_swap s5, s[4:7], 0xFFFFF
+// NOSICIVI: error: instruction not supported on this GPU
+// GFX10: s_buffer_atomic_swap s5, s[4:7], 0xfffff ; encoding: [0x42,0x01,0x00,0xf5,0xff,0xff,0x0f,0xfa]
+// GFX9: s_buffer_atomic_swap s5, s[4:7], 0xfffff ; encoding: [0x42,0x01,0x02,0xc1,0xff,0xff,0x0f,0x00]
+
+s_atc_probe 0x7, s[4:5], 0x1FFFFF
+// NOSICI: error: instruction not supported on this GPU
+// NOGFX9: error: expected a 21-bit signed offset
+// NOVI: error: expected a 20-bit unsigned offset
+
+s_atc_probe_buffer 0x1, s[8:11], 0x1FFFFF
+// NOSICI: error: instruction not supported on this GPU
+// NOGFX9: error: expected a 20-bit unsigned offset
+// NOVI: error: expected a 20-bit unsigned offset
+
+s_store_dword s1, s[2:3], 0x1FFFFF
+// NOSICI: error: instruction not supported on this GPU
+// NOGFX9: error: expected a 21-bit signed offset
+// NOVI: error: expected a 20-bit unsigned offset
+
+s_buffer_store_dword s10, s[92:95], 0x1FFFFF
+// NOSICI: error: instruction not supported on this GPU
+// NOGFX9: error: expected a 20-bit unsigned offset
+// NOVI: error: expected a 20-bit unsigned offset
+
+s_atomic_swap s5, s[2:3], 0x1FFFFF
+// NOSICIVI: error: instruction not supported on this GPU
+// NOGFX9: error: expected a 21-bit signed offset
+
+s_buffer_atomic_swap s5, s[4:7], 0x1FFFFF
+// NOSICIVI: error: instruction not supported on this GPU
+// NOGFX9: error: expected a 20-bit unsigned offset
+
+//===----------------------------------------------------------------------===//
+// Signed offsets (gfx9+)
+//===----------------------------------------------------------------------===//
+
+s_atc_probe 0x7, s[4:5], -1
+// NOVI: error: expected a 20-bit unsigned offset
+// GFX9: s_atc_probe 7, s[4:5], -0x1 ; encoding: [0xc2,0x01,0x9a,0xc0,0xff,0xff,0x1f,0x00]
+// GFX10: s_atc_probe 7, s[4:5], -0x1 ; encoding: [0xc2,0x01,0x98,0xf4,0xff,0xff,0x1f,0xfa]
+// NOSICI: error: instruction not supported on this GPU
+
+s_atc_probe_buffer 0x1, s[8:11], -1
+// NOVI: error: expected a 20-bit unsigned offset
+// NOSICI: error: instruction not supported on this GPU
+// NOGFX9: error: expected a 20-bit unsigned offset
+
+s_store_dword s1, s[2:3], -1
+// NOVI: error: expected a 20-bit unsigned offset
+// GFX9: s_store_dword s1, s[2:3], -0x1 ; encoding: [0x41,0x00,0x42,0xc0,0xff,0xff,0x1f,0x00]
+// GFX10: s_store_dword s1, s[2:3], -0x1 ; encoding: [0x41,0x00,0x40,0xf4,0xff,0xff,0x1f,0xfa]
+// NOSICI: error: instruction not supported on this GPU
+
+s_buffer_store_dword s10, s[92:95], -1
+// NOVI: error: expected a 20-bit unsigned offset
+// NOSICI: error: instruction not supported on this GPU
+// NOGFX9: error: expected a 20-bit unsigned offset
+
+s_load_dword s1, s[2:3], -1
+// NOVI: error: expected a 20-bit unsigned offset
+// GFX9: s_load_dword s1, s[2:3], -0x1 ; encoding: [0x41,0x00,0x02,0xc0,0xff,0xff,0x1f,0x00]
+// GFX10: s_load_dword s1, s[2:3], -0x1 ; encoding: [0x41,0x00,0x00,0xf4,0xff,0xff,0x1f,0xfa]
+// NOSICI: error: instruction not supported on this GPU
+
+s_buffer_load_dword s10, s[92:95], -1
+// NOVI: error: expected a 20-bit unsigned offset
+// NOSICI: error: instruction not supported on this GPU
+// NOGFX9: error: expected a 20-bit unsigned offset
+
+s_atomic_swap s5, s[2:3], -1
+// NOVI: error: instruction not supported on this GPU
+// GFX9: s_atomic_swap s5, s[2:3], -0x1 ; encoding: [0x41,0x01,0x02,0xc2,0xff,0xff,0x1f,0x00]
+// GFX10: s_atomic_swap s5, s[2:3], -0x1 ; encoding: [0x41,0x01,0x00,0xf6,0xff,0xff,0x1f,0xfa]
+// NOSICI: error: instruction not supported on this GPU
+
+s_buffer_atomic_swap s5, s[4:7], -1
+// NOVI: error: instruction not supported on this GPU
+// NOSICI: error: instruction not supported on this GPU
+// NOGFX9: error: expected a 20-bit unsigned offset
+
+s_atc_probe 0x7, s[4:5], 0xFFFFFFFFFFF00000
+// NOSICI: error: instruction not supported on this GPU
+// GFX10: s_atc_probe 7, s[4:5], -0x100000 ; encoding: [0xc2,0x01,0x98,0xf4,0x00,0x00,0x10,0xfa]
+// GFX9: s_atc_probe 7, s[4:5], -0x100000 ; encoding: [0xc2,0x01,0x9a,0xc0,0x00,0x00,0x10,0x00]
+// NOVI: error: expected a 20-bit unsigned offset
+
+s_atc_probe_buffer 0x1, s[8:11], 0xFFFFFFFFFFF00000
+// NOSICI: error: instruction not supported on this GPU
+// NOGFX9: error: expected a 20-bit unsigned offset
+// NOVI: error: expected a 20-bit unsigned offset
+
+s_store_dword s1, s[2:3], 0xFFFFFFFFFFF00000
+// NOSICI: error: instruction not supported on this GPU
+// GFX10: s_store_dword s1, s[2:3], -0x100000 ; encoding: [0x41,0x00,0x40,0xf4,0x00,0x00,0x10,0xfa]
+// GFX9: s_store_dword s1, s[2:3], -0x100000 ; encoding: [0x41,0x00,0x42,0xc0,0x00,0x00,0x10,0x00]
+// NOVI: error: expected a 20-bit unsigned offset
+
+s_buffer_store_dword s10, s[92:95], 0xFFFFFFFFFFF00000
+// NOSICI: error: instruction not supported on this GPU
+// NOGFX9: error: expected a 20-bit unsigned offset
+// NOVI: error: expected a 20-bit unsigned offset
+
+s_load_dword s1, s[2:3], 0xFFFFFFFFFFF00000
+// NOSICI: error: instruction not supported on this GPU
+// GFX10: s_load_dword s1, s[2:3], -0x100000 ; encoding: [0x41,0x00,0x00,0xf4,0x00,0x00,0x10,0xfa]
+// GFX9: s_load_dword s1, s[2:3], -0x100000 ; encoding: [0x41,0x00,0x02,0xc0,0x00,0x00,0x10,0x00]
+// NOVI: error: expected a 20-bit unsigned offset
+
+s_buffer_load_dword s10, s[92:95], 0xFFFFFFFFFFF00000
+// NOSICI: error: instruction not supported on this GPU
+// NOGFX9: error: expected a 20-bit unsigned offset
+// NOVI: error: expected a 20-bit unsigned offset
+
+s_atomic_swap s5, s[2:3], 0xFFFFFFFFFFF00000
+// NOSICIVI: error: instruction not supported on this GPU
+// GFX10: s_atomic_swap s5, s[2:3], -0x100000 ; encoding: [0x41,0x01,0x00,0xf6,0x00,0x00,0x10,0xfa]
+// GFX9: s_atomic_swap s5, s[2:3], -0x100000 ; encoding: [0x41,0x01,0x02,0xc2,0x00,0x00,0x10,0x00]
+
+s_buffer_atomic_swap s5, s[4:7], 0xFFFFFFFFFFF00000
+// NOSICIVI: error: instruction not supported on this GPU
+// NOGFX9: error: expected a 20-bit unsigned offset

llvm/test/MC/AMDGPU/smrd.s

 s_load_dword s1, s[2:3], 0xfffff
 // NOSI: error: instruction not supported on this GPU
 // CI: s_load_dword s1, s[2:3], 0xfffff ; encoding: [0xff,0x82,0x00,0xc0,0xff,0xff,0x0f,0x00]
 // VI: s_load_dword s1, s[2:3], 0xfffff ; encoding: [0x41,0x00,0x02,0xc0,0xff,0xff,0x0f,0x00]
 
 s_load_dword s1, s[2:3], 0x100000
 // NOSI: error: instruction not supported on this GPU
 // CI: s_load_dword s1, s[2:3], 0x100000 ; encoding: [0xff,0x82,0x00,0xc0,0x00,0x00,0x10,0x00]
-// NOVI: error: instruction not supported on this GPU
+// NOVI: error: expected a 20-bit unsigned offset
 
 s_load_dword s1, s[2:3], 0xffffffff
 // NOSI: error: instruction not supported on this GPU
 // CI: s_load_dword s1, s[2:3], 0xffffffff ; encoding: [0xff,0x82,0x00,0xc0,0xff,0xff,0xff,0xff]
-// NOVI: error: instruction not supported on this GPU
+// NOVI: error: expected a 20-bit unsigned offset
 
 //===----------------------------------------------------------------------===//
 // Instructions
 //===----------------------------------------------------------------------===//
 
 s_load_dword s1, s[2:3], 1
 // GCN: s_load_dword s1, s[2:3], 0x1 ; encoding: [0x01,0x83,0x00,0xc0]
 // VI:  s_load_dword s1, s[2:3], 0x1 ; encoding: [0x41,0x00,0x02,0xc0,0x01,0x00,0x00,0x00]

llvm/test/MC/Disassembler/AMDGPU/smem_gfx9.txt

 # GFX9: s_buffer_atomic_umin_x2 s[12:13], s[4:7], s0    ; encoding: [0x02,0x03,0x94,0xc1,0x00,0x00,0x00,0x00]
 0x02,0x03,0x94,0xc1,0x00,0x00,0x00,0x00
 
 # GFX9: s_buffer_atomic_xor s5, s[8:11], s0    ; encoding: [0x44,0x01,0x28,0xc1,0x00,0x00,0x00,0x00]
 0x44,0x01,0x28,0xc1,0x00,0x00,0x00,0x00
 
 # GFX9: s_buffer_atomic_xor_x2 s[100:101], s[4:7], s0    ; encoding: [0x02,0x19,0xa8,0xc1,0x00,0x00,0x00,0x00]
 0x02,0x19,0xa8,0xc1,0x00,0x00,0x00,0x00
+
+#===------------------------------------------------------------------------===#
+# Unsigned 20-bit offsets
+#===------------------------------------------------------------------------===#
+
+# GFX9: s_atc_probe 7, s[4:5], 0xfffff ; encoding: [0xc2,0x01,0x9a,0xc0,0xff,0xff,0x0f,0x00]
+0xc2,0x01,0x9a,0xc0,0xff,0xff,0x0f,0x00
+
+# GFX9: s_atc_probe_buffer 1, s[8:11], 0xfffff ; encoding: [0x44,0x00,0x9e,0xc0,0xff,0xff,0x0f,0x00]
+0x44,0x00,0x9e,0xc0,0xff,0xff,0x0f,0x00
+
+# GFX9: s_store_dword s1, s[2:3], 0xfffff ; encoding: [0x41,0x00,0x42,0xc0,0xff,0xff,0x0f,0x00]
+0x41,0x00,0x42,0xc0,0xff,0xff,0x0f,0x00
+
+# GFX9: s_buffer_store_dword s10, s[92:95], 0xfffff ; encoding: [0xae,0x02,0x62,0xc0,0xff,0xff,0x0f,0x00]
+0xae,0x02,0x62,0xc0,0xff,0xff,0x0f,0x00
+
+# GFX9: s_atomic_swap s5, s[2:3], 0xfffff ; encoding: [0x41,0x01,0x02,0xc2,0xff,0xff,0x0f,0x00]
+0x41,0x01,0x02,0xc2,0xff,0xff,0x0f,0x00
+
+# GFX9: s_buffer_atomic_swap s5, s[4:7], 0xfffff ; encoding: [0x42,0x01,0x02,0xc1,0xff,0xff,0x0f,0x00]
+0x42,0x01,0x02,0xc1,0xff,0xff,0x0f,0x00
+
+#===------------------------------------------------------------------------===#
+# Signed 20-bit offsets
+#===------------------------------------------------------------------------===#
+
+# GFX9: s_atc_probe 7, s[4:5], -0x1 ; encoding: [0xc2,0x01,0x9a,0xc0,0xff,0xff,0x1f,0x00]
+0xc2,0x01,0x9a,0xc0,0xff,0xff,0x1f,0x00
+
+# GFX9: s_store_dword s1, s[2:3], -0x1 ; encoding: [0x41,0x00,0x42,0xc0,0xff,0xff,0x1f,0x00]
+0x41,0x00,0x42,0xc0,0xff,0xff,0x1f,0x00
+
+# GFX9: s_load_dword s1, s[2:3], -0x1 ; encoding: [0x41,0x00,0x02,0xc0,0xff,0xff,0x1f,0x00]
+0x41,0x00,0x02,0xc0,0xff,0xff,0x1f,0x00
+
+# GFX9: s_atomic_swap s5, s[2:3], -0x1 ; encoding: [0x41,0x01,0x02,0xc2,0xff,0xff,0x1f,0x00]
+0x41,0x01,0x02,0xc2,0xff,0xff,0x1f,0x00
+
+# GFX9: s_atc_probe 7, s[4:5], -0x100000 ; encoding: [0xc2,0x01,0x9a,0xc0,0x00,0x00,0x10,0x00]
+0xc2,0x01,0x9a,0xc0,0x00,0x00,0x10,0x00
+
+# GFX9: s_store_dword s1, s[2:3], -0x100000 ; encoding: [0x41,0x00,0x42,0xc0,0x00,0x00,0x10,0x00]
+0x41,0x00,0x42,0xc0,0x00,0x00,0x10,0x00
+
+# GFX9: s_load_dword s1, s[2:3], -0x100000 ; encoding: [0x41,0x00,0x02,0xc0,0x00,0x00,0x10,0x00]
+0x41,0x00,0x02,0xc0,0x00,0x00,0x10,0x00
+
+# GFX9: s_atomic_swap s5, s[2:3], -0x100000 ; encoding: [0x41,0x01,0x02,0xc2,0x00,0x00,0x10,0x00]
+0x41,0x01,0x02,0xc2,0x00,0x00,0x10,0x00
+
+#===------------------------------------------------------------------------===#
+# Decoding of invalid buffer offsets
+#===------------------------------------------------------------------------===#
+
+# GFX9: s_atc_probe_buffer 1, s[8:11], -0x1 ; encoding: [0x44,0x00,0x9e,0xc0,0xff,0xff,0x1f,0x00]
+0x44,0x00,0x9e,0xc0,0xff,0xff,0x1f,0x00

llvm/test/MC/Disassembler/AMDGPU/smem_vi.txt

 # VI: s_atc_probe 0, s[4:5], 0x0 ; encoding: [0x02,0x00,0x9a,0xc0,0x00,0x00,0x00,0x00]
 0x02,0x00,0x9a,0xc0,0x00,0x00,0x00,0x00
 
 # VI: s_atc_probe_buffer 1, s[8:11], s0 ; encoding: [0x44,0x00,0x9c,0xc0,0x00,0x00,0x00,0x00]
 0x44,0x00,0x9c,0xc0,0x00,0x00,0x00,0x00
 
 # VI: s_atc_probe_buffer 0, s[8:11], s101 ; encoding: [0x04,0x00,0x9c,0xc0,0x65,0x00,0x00,0x00]
 0x04,0x00,0x9c,0xc0,0x65,0x00,0x00,0x00
+
+#===------------------------------------------------------------------------===#
+# Unsigned 20-bit offsets
+#===------------------------------------------------------------------------===#
+
+# VI: s_atc_probe 7, s[4:5], 0xfffff ; encoding: [0xc2,0x01,0x9a,0xc0,0xff,0xff,0x0f,0x00]
+0xc2,0x01,0x9a,0xc0,0xff,0xff,0x0f,0x00
+
+# VI: s_atc_probe_buffer 1, s[8:11], 0xfffff ; encoding: [0x44,0x00,0x9e,0xc0,0xff,0xff,0x0f,0x00]
+0x44,0x00,0x9e,0xc0,0xff,0xff,0x0f,0x00
+
+# VI: s_load_dword s1, s[2:3], 0xfffff ; encoding: [0x41,0x00,0x02,0xc0,0xff,0xff,0x0f,0x00]
+0x41,0x00,0x02,0xc0,0xff,0xff,0x0f,0x00
+
+# VI: s_store_dword s1, s[2:3], 0xfffff ; encoding: [0x41,0x00,0x42,0xc0,0xff,0xff,0x0f,0x00]
+0x41,0x00,0x42,0xc0,0xff,0xff,0x0f,0x00
+
+# VI: s_buffer_store_dword s10, s[92:95], 0xfffff ; encoding: [0xae,0x02,0x62,0xc0,0xff,0xff,0x0f,0x00]
+0xae,0x02,0x62,0xc0,0xff,0xff,0x0f,0x00
+
+#===------------------------------------------------------------------------===#
+# Decoding of invalid (21-bit) buffer offsets
+#===------------------------------------------------------------------------===#
+
+# VI: s_store_dword s1, s[2:3], 0xfffff ; encoding: [0x41,0x00,0x42,0xc0,0xff,0xff,0x0f,0x00]
+0x41,0x00,0x42,0xc0,0xff,0xff,0x1f,0x00
+
+# VI: s_buffer_store_dword s10, s[92:95], 0xfffff ; encoding: [0xae,0x02,0x62,0xc0,0xff,0xff,0x0f,0x00]
+0xae,0x02,0x62,0xc0,0xff,0xff,0x1f,0x00