[AMDGPU][MC] Added validation of d16 and r128 modifiers of MIMG opcodes

[dp]

Summary

r128 dropped in GFX9;
d16 is supported by GFX8/9 only.

See bugs 36094 and 36095:

https://bugs.llvm.org/show_bug.cgi?id=36094
https://bugs.llvm.org/show_bug.cgi?id=36095

[arsenm]

lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

2344 ↗

(On Diff #131980)

I don't like this listing of generations. We should add a new r128 subtarget feature bit instead

[dp]

Added R128 feature as suggested by Matt

[artem.tamazov]

lib/Target/AMDGPU/AMDGPU.td

35 ↗

(On Diff #132349)

R128 seems too cryptic. For example, I would decode "R128" as "Support for register quads", which is of course wrong.

lib/Target/AMDGPU/AMDGPUSubtarget.h

142 ↗

(On Diff #132349)

Ditto.

[dp]

lib/Target/AMDGPU/AMDGPU.td

35 ↗

(On Diff #132349)

We may introduce a longer name, e.g. ShortTextureResource, but I think a reader with good knowledge of AMD GPU would be curious if the name stands for R128 and what is the difference.

[arsenm]

lib/Target/AMDGPU/AMDGPU.td

35 ↗

(On Diff #132349)

I think mimg-r128 would be a good name.

[dp]

Renamed R128 to MIMG_R128 as suggested by Atrem and Matt

[dp]

Corrected a typo in feature name

[arsenm]

lib/Target/AMDGPU/AMDGPU.td

34 ↗

(On Diff #132436)

Subtarget feature names should be all lowercase

[dp]

Corrected subtarget feature name.

llvm/trunk/lib/Target/AMDGPU/AMDGPU.td

 >;
 
 def FeatureFastFMAF32 : SubtargetFeature<"fast-fmaf",
   "FastFMAF32",
   "true",
   "Assuming f32 fma is at least as fast as mul + add"
 >;
 
+def FeatureMIMG_R128 : SubtargetFeature<"mimg-r128",
+  "MIMG_R128",
+  "true",
+  "Support 128-bit texture resources"
+>;
+
 def HalfRate64Ops : SubtargetFeature<"half-rate-64-ops",
   "HalfRate64Ops",
   "true",
   "Most fp64 instructions are half rate instead of quarter"
 >;
 
 def FeatureR600ALUInst : SubtargetFeature<"R600ALUInst",
   "R600ALUInst",

 >;
 
 def FeatureNorthernIslands : SubtargetFeatureGeneration<"NORTHERN_ISLANDS",
   [FeatureFetchLimit16, FeatureWavefrontSize64,
    FeatureLocalMemorySize32768]
 >;
 
 def FeatureSouthernIslands : SubtargetFeatureGeneration<"SOUTHERN_ISLANDS",
-  [FeatureFP64, FeatureLocalMemorySize32768,
+  [FeatureFP64, FeatureLocalMemorySize32768, FeatureMIMG_R128,
   FeatureWavefrontSize64, FeatureGCN,
   FeatureLDSBankCount32, FeatureMovrel]
 >;
 
 def FeatureSeaIslands : SubtargetFeatureGeneration<"SEA_ISLANDS",
-  [FeatureFP64, FeatureLocalMemorySize65536,
+  [FeatureFP64, FeatureLocalMemorySize65536, FeatureMIMG_R128,
   FeatureWavefrontSize64, FeatureGCN, FeatureFlatAddressSpace,
   FeatureCIInsts, FeatureMovrel]
 >;
 
 def FeatureVolcanicIslands : SubtargetFeatureGeneration<"VOLCANIC_ISLANDS",
-  [FeatureFP64, FeatureLocalMemorySize65536,
+  [FeatureFP64, FeatureLocalMemorySize65536, FeatureMIMG_R128,
    FeatureWavefrontSize64, FeatureFlatAddressSpace, FeatureGCN,
    FeatureGCN3Encoding, FeatureCIInsts, Feature16BitInsts,
    FeatureSMemRealTime, FeatureVGPRIndexMode, FeatureMovrel,
    FeatureScalarStores, FeatureInv2PiInlineImm,
    FeatureSDWA, FeatureSDWAOutModsVOPC, FeatureSDWAMac, FeatureDPP,
    FeatureIntClamp
   ]
 >;

llvm/trunk/lib/Target/AMDGPU/AMDGPUSubtarget.h

protected:
   bool EnableLoadStoreOpt;
   bool EnableUnsafeDSOffsetFolding;
   bool EnableSIScheduler;
   bool DumpCode;
 
   // Subtarget statically properties set by tablegen
   bool FP64;
   bool FMA;
+  bool MIMG_R128;
   bool IsGCN;
   bool GCN3Encoding;
   bool CIInsts;
   bool GFX9Insts;
   bool SGPRInitBug;
   bool HasSMemRealTime;
   bool Has16BitInsts;
   bool HasIntClamp;
public:
   bool hasVOP3PInsts() const {
     return HasVOP3PInsts;
   }
 
   bool hasFP64() const {
     return FP64;
   }
 
+  bool hasMIMG_R128() const {
+    return MIMG_R128;
+  }
+
   bool hasFastFMAF32() const {
     return FastFMAF32;
   }
 
   bool hasHalfRate64Ops() const {
     return HalfRate64Ops;
   }
 

llvm/trunk/lib/Target/AMDGPU/AMDGPUSubtarget.cpp

: AMDGPUGenSubtargetInfo(TT, GPU, FS),
     EnablePromoteAlloca(false),
     EnableLoadStoreOpt(false),
     EnableUnsafeDSOffsetFolding(false),
     EnableSIScheduler(false),
     DumpCode(false),
 
     FP64(false),
     FMA(false),
+    MIMG_R128(false),
     IsGCN(false),
     GCN3Encoding(false),
     CIInsts(false),
     GFX9Insts(false),
     SGPRInitBug(false),
     HasSMemRealTime(false),
     Has16BitInsts(false),
     HasIntClamp(false),

llvm/trunk/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
     }
     KernelScope.initialize(getContext());
   }
 
   bool hasXNACK() const {
     return AMDGPU::hasXNACK(getSTI());
   }
 
+  bool hasMIMG_R128() const {
+    return AMDGPU::hasMIMG_R128(getSTI());
+  }
+
   bool isSI() const {
     return AMDGPU::isSI(getSTI());
   }
 
   bool isCI() const {
     return AMDGPU::isCI(getSTI());
   }
 
private:
   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
 
   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc);
   bool validateConstantBusLimitations(const MCInst &Inst);
   bool validateEarlyClobberLimitations(const MCInst &Inst);
   bool validateIntClampSupported(const MCInst &Inst);
   bool validateMIMGAtomicDMask(const MCInst &Inst);
   bool validateMIMGDataSize(const MCInst &Inst);
+  bool validateMIMGR128(const MCInst &Inst);
+  bool validateMIMGD16(const MCInst &Inst);
   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
 
   bool trySkipId(const StringRef Id);
   bool trySkipToken(const AsmToken::TokenKind Kind);
   bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
   bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
bool AMDGPUAsmParser::validateMIMGAtomicDMask(const MCInst &Inst) {
 
   // This is an incomplete check because image_atomic_cmpswap
   // may only use 0x3 and 0xf while other atomic operations
   // may use 0x1 and 0x3. However these limitations are
   // verified when we check that dmask matches dst size.
   return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
 }
 
+bool AMDGPUAsmParser::validateMIMGR128(const MCInst &Inst) {
+
+  const unsigned Opc = Inst.getOpcode();
+  const MCInstrDesc &Desc = MII.get(Opc);
+
+  if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
+    return true;
+
+  int Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::r128);
+  assert(Idx != -1);
+
+  bool R128 = (Inst.getOperand(Idx).getImm() != 0);
+
+  return !R128 || hasMIMG_R128();
+}
+
+bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) {
+
+  const unsigned Opc = Inst.getOpcode();
+  const MCInstrDesc &Desc = MII.get(Opc);
+
+  if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
+    return true;
+  if ((Desc.TSFlags & SIInstrFlags::D16) == 0)
+    return true;
+
+  return !isCI() && !isSI();
+}
+
 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
                                           const SMLoc &IDLoc) {
   if (!validateConstantBusLimitations(Inst)) {
     Error(IDLoc,
       "invalid operand (violates constant bus restrictions)");
     return false;
   }
   if (!validateEarlyClobberLimitations(Inst)) {
Error(IDLoc,
       "image data size does not match dmask and tfe");
     return false;
   }
   if (!validateMIMGAtomicDMask(Inst)) {
     Error(IDLoc,
       "invalid atomic image dmask");
     return false;
   }
+  if (!validateMIMGR128(Inst)) {
+    Error(IDLoc,
+      "r128 modifier is not supported on this GPU");
+    return false;
+  }
+  // For MUBUF/MTBUF d16 is a part of opcode, so there is nothing to validate.
+  if (!validateMIMGD16(Inst)) {
+    Error(IDLoc,
+      "d16 modifier is not supported on this GPU");
+    return false;
+  }
 
   return true;
 }
 
 static std::string AMDGPUMnemonicSpellCheck(StringRef S, uint64_t FBS,
                                             unsigned VariantID = 0);
 
 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,

llvm/trunk/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h

case CallingConv::SPIR_KERNEL:
     return true;
   default:
     return false;
   }
 }
 
 bool hasXNACK(const MCSubtargetInfo &STI);
 
+bool hasMIMG_R128(const MCSubtargetInfo &STI);
+
 bool isSI(const MCSubtargetInfo &STI);
 bool isCI(const MCSubtargetInfo &STI);
 bool isVI(const MCSubtargetInfo &STI);
 bool isGFX9(const MCSubtargetInfo &STI);
 
 /// \brief Is Reg - scalar register
 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI);
 

llvm/trunk/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

default:
     return false;
   }
 }
 
 bool hasXNACK(const MCSubtargetInfo &STI) {
   return STI.getFeatureBits()[AMDGPU::FeatureXNACK];
 }
 
+bool hasMIMG_R128(const MCSubtargetInfo &STI) {
+  return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128];
+}
+
 bool isSI(const MCSubtargetInfo &STI) {
   return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands];
 }
 
 bool isCI(const MCSubtargetInfo &STI) {
   return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands];
 }
 

llvm/trunk/test/MC/AMDGPU/mimg.s

-// RUN: llvm-mc -arch=amdgcn -show-encoding %s | FileCheck %s --check-prefix=SICI
-// RUN: llvm-mc -arch=amdgcn -mcpu=tahiti -show-encoding %s | FileCheck %s --check-prefix=SICI
-// RUN: llvm-mc -arch=amdgcn -mcpu=fiji -show-encoding %s | FileCheck %s --check-prefix=VI
+// RUN: not llvm-mc -arch=amdgcn -show-encoding %s | FileCheck %s --check-prefix=GCN --check-prefix=SICI --check-prefix=SICIVI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=tahiti -show-encoding %s | FileCheck %s --check-prefix=GCN --check-prefix=SICI --check-prefix=SICIVI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=bonaire -show-encoding %s | FileCheck %s --check-prefix=GCN --check-prefix=SICI --check-prefix=SICIVI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx900 -show-encoding %s | FileCheck %s --check-prefix=GCN --check-prefix=GFX9 --check-prefix=GFX89
+
+// RUN:     llvm-mc -arch=amdgcn -mcpu=fiji -show-encoding %s | FileCheck %s --check-prefix=GCN  --check-prefix=SICIVI --check-prefix=VI --check-prefix=GFX89
+
+// RUN: not llvm-mc -arch=amdgcn -show-encoding %s 2>&1 | FileCheck %s --check-prefix=NOSICI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=tahiti -show-encoding %s 2>&1 | FileCheck %s --check-prefix=NOSICI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=bonaire -show-encoding %s 2>&1 | FileCheck %s --check-prefix=NOSICI
+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx900 -show-encoding %s 2>&1 | FileCheck %s --check-prefix=NOGFX9
 
 //===----------------------------------------------------------------------===//
 // Image Load/Store
 //===----------------------------------------------------------------------===//
 
 image_load    v[4:6], v[237:240], s[28:35] dmask:0x7 unorm
-// SICI: image_load v[4:6], v[237:240], s[28:35] dmask:0x7 unorm ; encoding: [0x00,0x17,0x00,0xf0,0xed,0x04,0x07,0x00]
-// VI:   image_load v[4:6], v[237:240], s[28:35] dmask:0x7 unorm ; encoding: [0x00,0x17,0x00,0xf0,0xed,0x04,0x07,0x00]
+// GCN:  image_load v[4:6], v[237:240], s[28:35] dmask:0x7 unorm ; encoding: [0x00,0x17,0x00,0xf0,0xed,0x04,0x07,0x00]
 
 image_load    v4, v[237:240], s[28:35]
-// SICI: image_load v4, v[237:240], s[28:35] ; encoding: [0x00,0x00,0x00,0xf0,0xed,0x04,0x07,0x00]
-// VI:   image_load v4, v[237:240], s[28:35] ; encoding: [0x00,0x00,0x00,0xf0,0xed,0x04,0x07,0x00]
+// GCN:  image_load v4, v[237:240], s[28:35] ; encoding: [0x00,0x00,0x00,0xf0,0xed,0x04,0x07,0x00]
 
 image_load    v[4:7], v[237:240], s[28:35] dmask:0x7 tfe
-// SICI: image_load v[4:7], v[237:240], s[28:35] dmask:0x7 tfe ; encoding: [0x00,0x07,0x01,0xf0,0xed,0x04,0x07,0x00]
-// VI:   image_load v[4:7], v[237:240], s[28:35] dmask:0x7 tfe ; encoding: [0x00,0x07,0x01,0xf0,0xed,0x04,0x07,0x00]
+// GCN:  image_load v[4:7], v[237:240], s[28:35] dmask:0x7 tfe ; encoding: [0x00,0x07,0x01,0xf0,0xed,0x04,0x07,0x00]
 
 // Verify support of all possible modifiers.
 // FIXME: This test is incorrect because r128 assumes a 128-bit SRSRC.
-// FIXME: Check that d16 is not supported before VI
 image_load    v[5:6], v[1:4], s[8:15] dmask:0x1 unorm glc slc r128 tfe lwe da d16
-// VI:   image_load v[5:6], v[1:4], s[8:15] dmask:0x1 unorm glc slc r128 tfe lwe da d16 ; encoding: [0x00,0xf1,0x03,0xf2,0x01,0x05,0x02,0x80]
+// NOSICI: error: d16 modifier is not supported on this GPU
+// VI:     image_load v[5:6], v[1:4], s[8:15] dmask:0x1 unorm glc slc r128 tfe lwe da d16 ; encoding: [0x00,0xf1,0x03,0xf2,0x01,0x05,0x02,0x80]
+// NOGFX9: error: r128 modifier is not supported on this GPU
 
-// FIXME: Check that d16 is not supported before VI
 image_load v5, v[1:4], s[8:15] d16
-// VI:   image_load v5, v[1:4], s[8:15] d16 ; encoding: [0x00,0x00,0x00,0xf0,0x01,0x05,0x02,0x80]
+// NOSICI: error: d16 modifier is not supported on this GPU
+// GFX89:  image_load v5, v[1:4], s[8:15] d16 ; encoding: [0x00,0x00,0x00,0xf0,0x01,0x05,0x02,0x80]
+
+image_load v5, v[1:4], s[8:15] r128
+// SICIVI: image_load v5, v[1:4], s[8:15] r128 ; encoding: [0x00,0x80,0x00,0xf0,0x01,0x05,0x02,0x00]
+// NOGFX9: error: r128 modifier is not supported on this GPU
 
 image_store   v[193:195], v[237:240], s[28:35] dmask:0x7 unorm
-// SICI: image_store v[193:195], v[237:240], s[28:35] dmask:0x7 unorm ; encoding: [0x00,0x17,0x20,0xf0,0xed,0xc1,0x07,0x00]
-// VI:   image_store v[193:195], v[237:240], s[28:35] dmask:0x7 unorm ; encoding: [0x00,0x17,0x20,0xf0,0xed,0xc1,0x07,0x00]
+// GCN: image_store v[193:195], v[237:240], s[28:35] dmask:0x7 unorm ; encoding: [0x00,0x17,0x20,0xf0,0xed,0xc1,0x07,0x00]
 
 image_store   v193, v[237:240], s[28:35]
-// SICI: image_store v193, v[237:240], s[28:35] ; encoding: [0x00,0x00,0x20,0xf0,0xed,0xc1,0x07,0x00]
-// VI:   image_store v193, v[237:240], s[28:35] ; encoding: [0x00,0x00,0x20,0xf0,0xed,0xc1,0x07,0x00]
+// GCN: image_store v193, v[237:240], s[28:35] ; encoding: [0x00,0x00,0x20,0xf0,0xed,0xc1,0x07,0x00]
 
 image_store   v[193:194], v[237:240], s[28:35] tfe
-// SICI: image_store v[193:194], v[237:240], s[28:35] tfe ; encoding: [0x00,0x00,0x21,0xf0,0xed,0xc1,0x07,0x00]
-// VI:   image_store v[193:194], v[237:240], s[28:35] tfe ; encoding: [0x00,0x00,0x21,0xf0,0xed,0xc1,0x07,0x00]
+// GCN: image_store v[193:194], v[237:240], s[28:35] tfe ; encoding: [0x00,0x00,0x21,0xf0,0xed,0xc1,0x07,0x00]
 
 // Verify support of all possible modifiers.
 // FIXME: This test is incorrect because r128 assumes a 128-bit SRSRC.
-// FIXME: Check that d16 is not supported before VI
 image_store   v5, v[1:4], s[8:15] dmask:0x1 unorm glc slc r128 lwe da d16
-// VI:   image_store v5, v[1:4], s[8:15] dmask:0x1 unorm glc slc r128 lwe da d16 ; encoding: [0x00,0xf1,0x22,0xf2,0x01,0x05,0x02,0x80]
+// NOSICI: error: d16 modifier is not supported on this GPU
+// VI:     image_store v5, v[1:4], s[8:15] dmask:0x1 unorm glc slc r128 lwe da d16 ; encoding: [0x00,0xf1,0x22,0xf2,0x01,0x05,0x02,0x80]
+// NOGFX9: error: r128 modifier is not supported on this GPU
 
-// FIXME: Check that d16 is not supported before VI
 image_store    v5, v[1:4], s[8:15] d16
-// VI:   image_store v5, v[1:4], s[8:15] d16 ; encoding: [0x00,0x00,0x20,0xf0,0x01,0x05,0x02,0x80]
+// NOSICI: error: d16 modifier is not supported on this GPU
+// GFX89:  image_store v5, v[1:4], s[8:15] d16 ; encoding: [0x00,0x00,0x20,0xf0,0x01,0x05,0x02,0x80]
+
+// FIXME: This test is incorrect because r128 assumes a 128-bit SRSRC.
+image_store    v5, v[1:4], s[8:15] r128
+// SICIVI: image_store v5, v[1:4], s[8:15] r128 ; encoding: [0x00,0x80,0x20,0xf0,0x01,0x05,0x02,0x00]
+// NOGFX9: error: r128 modifier is not supported on this GPU
 
 //===----------------------------------------------------------------------===//
 // Image Sample
 //===----------------------------------------------------------------------===//
 
 image_sample  v[193:195], v[237:240], s[28:35], s[4:7] dmask:0x7 unorm
-// SICI: image_sample v[193:195], v[237:240], s[28:35], s[4:7] dmask:0x7 unorm ; encoding: [0x00,0x17,0x80,0xf0,0xed,0xc1,0x27,0x00]
-// VI:   image_sample v[193:195], v[237:240], s[28:35], s[4:7] dmask:0x7 unorm ; encoding: [0x00,0x17,0x80,0xf0,0xed,0xc1,0x27,0x00]
+// GCN: image_sample v[193:195], v[237:240], s[28:35], s[4:7] dmask:0x7 unorm ; encoding: [0x00,0x17,0x80,0xf0,0xed,0xc1,0x27,0x00]
 
 image_sample  v193, v[237:240], s[28:35], s[4:7]
-// SICI: image_sample v193, v[237:240], s[28:35], s[4:7] ; encoding: [0x00,0x00,0x80,0xf0,0xed,0xc1,0x27,0x00]
-// VI:   image_sample v193, v[237:240], s[28:35], s[4:7] ; encoding: [0x00,0x00,0x80,0xf0,0xed,0xc1,0x27,0x00]
+// GCN: image_sample v193, v[237:240], s[28:35], s[4:7] ; encoding: [0x00,0x00,0x80,0xf0,0xed,0xc1,0x27,0x00]
 
 image_sample  v[193:194], v[237:240], s[28:35], s[4:7] tfe
-// SICI: image_sample v[193:194], v[237:240], s[28:35], s[4:7] tfe ; encoding: [0x00,0x00,0x81,0xf0,0xed,0xc1,0x27,0x00]
-// VI:   image_sample v[193:194], v[237:240], s[28:35], s[4:7] tfe ; encoding: [0x00,0x00,0x81,0xf0,0xed,0xc1,0x27,0x00]
+// GCN: image_sample v[193:194], v[237:240], s[28:35], s[4:7] tfe ; encoding: [0x00,0x00,0x81,0xf0,0xed,0xc1,0x27,0x00]
+
+// FIXME: This test is incorrect because r128 assumes a 128-bit SRSRC.
+image_sample  v193, v[237:240], s[28:35], s[4:7] r128
+// SICIVI: image_sample v193, v[237:240], s[28:35], s[4:7] r128 ; encoding: [0x00,0x80,0x80,0xf0,0xed,0xc1,0x27,0x00]
+// NOGFX9: error: r128 modifier is not supported on this GPU
+
+image_sample  v193, v[237:240], s[28:35], s[4:7] d16
+// NOSICI: error: d16 modifier is not supported on this GPU
+// GFX89:  image_sample v193, v[237:240], s[28:35], s[4:7] d16 ; encoding: [0x00,0x00,0x80,0xf0,0xed,0xc1,0x27,0x80]
 
 //===----------------------------------------------------------------------===//
 // Image Atomics
 //===----------------------------------------------------------------------===//
 
 image_atomic_add v4, v[192:195], s[28:35] dmask:0x1 unorm glc
-// SICI: image_atomic_add v4, v[192:195], s[28:35] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x44,0xf0,0xc0,0x04,0x07,0x00]
-// VI:   image_atomic_add v4, v[192:195], s[28:35] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x48,0xf0,0xc0,0x04,0x07,0x00]
+// SICI:  image_atomic_add v4, v[192:195], s[28:35] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x44,0xf0,0xc0,0x04,0x07,0x00]
+// GFX89: image_atomic_add v4, v[192:195], s[28:35] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x48,0xf0,0xc0,0x04,0x07,0x00]
 
 image_atomic_add v252, v2, s[8:15] dmask:0x1 unorm
-// SICI: image_atomic_add v252, v2, s[8:15] dmask:0x1 unorm ; encoding: [0x00,0x11,0x44,0xf0,0x02,0xfc,0x02,0x00]
-// VI:   image_atomic_add v252, v2, s[8:15] dmask:0x1 unorm ; encoding: [0x00,0x11,0x48,0xf0,0x02,0xfc,0x02,0x00]
+// SICI:  image_atomic_add v252, v2, s[8:15] dmask:0x1 unorm ; encoding: [0x00,0x11,0x44,0xf0,0x02,0xfc,0x02,0x00]
+// GFX89: image_atomic_add v252, v2, s[8:15] dmask:0x1 unorm ; encoding: [0x00,0x11,0x48,0xf0,0x02,0xfc,0x02,0x00]
 
 image_atomic_add v[6:7], v255, s[8:15] dmask:0x3
-// SICI: image_atomic_add v[6:7], v255, s[8:15] dmask:0x3 ; encoding: [0x00,0x03,0x44,0xf0,0xff,0x06,0x02,0x00]
-// VI:   image_atomic_add v[6:7], v255, s[8:15] dmask:0x3 ; encoding: [0x00,0x03,0x48,0xf0,0xff,0x06,0x02,0x00]
+// SICI:  image_atomic_add v[6:7], v255, s[8:15] dmask:0x3 ; encoding: [0x00,0x03,0x44,0xf0,0xff,0x06,0x02,0x00]
+// GFX89: image_atomic_add v[6:7], v255, s[8:15] dmask:0x3 ; encoding: [0x00,0x03,0x48,0xf0,0xff,0x06,0x02,0x00]
 
 image_atomic_add v7, v3, s[0:7] dmask:0x1 glc
-// SICI: image_atomic_add v7, v3, s[0:7] dmask:0x1 glc ; encoding: [0x00,0x21,0x44,0xf0,0x03,0x07,0x00,0x00]
-// VI:   image_atomic_add v7, v3, s[0:7] dmask:0x1 glc ; encoding: [0x00,0x21,0x48,0xf0,0x03,0x07,0x00,0x00]
+// SICI:  image_atomic_add v7, v3, s[0:7] dmask:0x1 glc ; encoding: [0x00,0x21,0x44,0xf0,0x03,0x07,0x00,0x00]
+// GFX89: image_atomic_add v7, v3, s[0:7] dmask:0x1 glc ; encoding: [0x00,0x21,0x48,0xf0,0x03,0x07,0x00,0x00]
 
 image_atomic_add v8, v4, s[8:15] dmask:0x1 slc
-// SICI: image_atomic_add v8, v4, s[8:15] dmask:0x1 slc ; encoding: [0x00,0x01,0x44,0xf2,0x04,0x08,0x02,0x00]
-// VI:   image_atomic_add v8, v4, s[8:15] dmask:0x1 slc ; encoding: [0x00,0x01,0x48,0xf2,0x04,0x08,0x02,0x00]
+// SICI:  image_atomic_add v8, v4, s[8:15] dmask:0x1 slc ; encoding: [0x00,0x01,0x44,0xf2,0x04,0x08,0x02,0x00]
+// GFX89: image_atomic_add v8, v4, s[8:15] dmask:0x1 slc ; encoding: [0x00,0x01,0x48,0xf2,0x04,0x08,0x02,0x00]
 
 image_atomic_add v9, v5, s[8:15] dmask:0x1 unorm glc slc lwe da
-// SICI: image_atomic_add v9, v5, s[8:15] dmask:0x1 unorm glc slc lwe da ; encoding: [0x00,0x71,0x46,0xf2,0x05,0x09,0x02,0x00]
-// VI:   image_atomic_add v9, v5, s[8:15] dmask:0x1 unorm glc slc lwe da ; encoding: [0x00,0x71,0x4a,0xf2,0x05,0x09,0x02,0x00]
+// SICI:  image_atomic_add v9, v5, s[8:15] dmask:0x1 unorm glc slc lwe da ; encoding: [0x00,0x71,0x46,0xf2,0x05,0x09,0x02,0x00]
+// GFX89: image_atomic_add v9, v5, s[8:15] dmask:0x1 unorm glc slc lwe da ; encoding: [0x00,0x71,0x4a,0xf2,0x05,0x09,0x02,0x00]
 
 image_atomic_add v10, v6, s[8:15] dmask:0x1 lwe
-// SICI: image_atomic_add v10, v6, s[8:15] dmask:0x1 lwe ; encoding: [0x00,0x01,0x46,0xf0,0x06,0x0a,0x02,0x00]
-// VI:   image_atomic_add v10, v6, s[8:15] dmask:0x1 lwe ; encoding: [0x00,0x01,0x4a,0xf0,0x06,0x0a,0x02,0x00]
+// SICI:  image_atomic_add v10, v6, s[8:15] dmask:0x1 lwe ; encoding: [0x00,0x01,0x46,0xf0,0x06,0x0a,0x02,0x00]
+// GFX89: image_atomic_add v10, v6, s[8:15] dmask:0x1 lwe ; encoding: [0x00,0x01,0x4a,0xf0,0x06,0x0a,0x02,0x00]
 
 image_atomic_add v11, v7, s[8:15] dmask:0x1 da
-// SICI: image_atomic_add v11, v7, s[8:15] dmask:0x1 da ; encoding: [0x00,0x41,0x44,0xf0,0x07,0x0b,0x02,0x00]
-// VI:   image_atomic_add v11, v7, s[8:15] dmask:0x1 da ; encoding: [0x00,0x41,0x48,0xf0,0x07,0x0b,0x02,0x00]
+// SICI:  image_atomic_add v11, v7, s[8:15] dmask:0x1 da ; encoding: [0x00,0x41,0x44,0xf0,0x07,0x0b,0x02,0x00]
+// GFX89: image_atomic_add v11, v7, s[8:15] dmask:0x1 da ; encoding: [0x00,0x41,0x48,0xf0,0x07,0x0b,0x02,0x00]
 
 image_atomic_swap v4, v[192:195], s[28:35] dmask:0x1 unorm glc
-// SICI: image_atomic_swap v4, v[192:195], s[28:35] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x3c,0xf0,0xc0,0x04,0x07,0x00]
-// VI:   image_atomic_swap v4, v[192:195], s[28:35] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x40,0xf0,0xc0,0x04,0x07,0x00]
+// SICI:  image_atomic_swap v4, v[192:195], s[28:35] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x3c,0xf0,0xc0,0x04,0x07,0x00]
+// GFX89: image_atomic_swap v4, v[192:195], s[28:35] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x40,0xf0,0xc0,0x04,0x07,0x00]
 
 image_atomic_cmpswap v[4:5], v[192:195], s[28:35] dmask:0x3 unorm glc
-// SICI: image_atomic_cmpswap v[4:5], v[192:195], s[28:35] dmask:0x3 unorm glc ; encoding: [0x00,0x33,0x40,0xf0,0xc0,0x04,0x07,0x00]
-// VI:   image_atomic_cmpswap v[4:5], v[192:195], s[28:35] dmask:0x3 unorm glc ; encoding: [0x00,0x33,0x44,0xf0,0xc0,0x04,0x07,0x00]
+// SICI:  image_atomic_cmpswap v[4:5], v[192:195], s[28:35] dmask:0x3 unorm glc ; encoding: [0x00,0x33,0x40,0xf0,0xc0,0x04,0x07,0x00]
+// GFX89: image_atomic_cmpswap v[4:5], v[192:195], s[28:35] dmask:0x3 unorm glc ; encoding: [0x00,0x33,0x44,0xf0,0xc0,0x04,0x07,0x00]
 
 image_atomic_cmpswap v[4:7], v[192:195], s[28:35] dmask:0xf unorm glc
-// SICI: image_atomic_cmpswap v[4:7], v[192:195], s[28:35] dmask:0xf unorm glc ; encoding: [0x00,0x3f,0x40,0xf0,0xc0,0x04,0x07,0x00]
-// VI:   image_atomic_cmpswap v[4:7], v[192:195], s[28:35] dmask:0xf unorm glc ; encoding: [0x00,0x3f,0x44,0xf0,0xc0,0x04,0x07,0x00]
+// SICI:  image_atomic_cmpswap v[4:7], v[192:195], s[28:35] dmask:0xf unorm glc ; encoding: [0x00,0x3f,0x40,0xf0,0xc0,0x04,0x07,0x00]
+// GFX89: image_atomic_cmpswap v[4:7], v[192:195], s[28:35] dmask:0xf unorm glc ; encoding: [0x00,0x3f,0x44,0xf0,0xc0,0x04,0x07,0x00]
+
+// FIXME: This test is incorrect because r128 assumes a 128-bit SRSRC.
+image_atomic_add v10, v6, s[8:15] dmask:0x1 r128
+// SICI: image_atomic_add v10, v6, s[8:15] dmask:0x1 r128 ; encoding: [0x00,0x81,0x44,0xf0,0x06,0x0a,0x02,0x00]
+// VI:   image_atomic_add v10, v6, s[8:15] dmask:0x1 r128 ; encoding: [0x00,0x81,0x48,0xf0,0x06,0x0a,0x02,0x00]
+// NOGFX9: error: r128 modifier is not supported on this GPU