[AMDGPU][MC] Added support of 64-bit image atomics

[dp]

Summary

See bug 35998: https://bugs.llvm.org/show_bug.cgi?id=35998

It is not clear if atomics should support dmask=0. Currently only 0x1, 0x3 and 0xf values are allowed. Spec states that "all other values of dmask are illegal".

Depends on D42186

[dp]

Updated to exclude changes from D42186

[artem.tamazov]

test/MC/AMDGPU/mimg.s

37 ↗

(On Diff #131232)

Does SP3 supporting this syntax? SP3 docs says that all image_atomic insts have ...vgpr_d[4], vgpr_a..., i.e. 4-dword data and 1-dword address.

[dp]

test/MC/AMDGPU/mimg.s

37 ↗

(On Diff #131232)

Available SP3 documentation looks incomplete. Here are a few examples of code I got from SP3 decoder:

// default SP3 dmask is 1
[0x00,0x11,0x48,0xf0,0x01,0x05,0x17,0x00] image_atomic_add v5, v1, s[92:99] unorm
[0x00,0x13,0x48,0xf0,0x01,0x05,0x02,0x00] image_atomic_add v[5:6], v1, s[8:15] dmask:0x3 unorm

Also for gfx10 SP3 decoder may generate address with more than 1 dword.

llvm/trunk/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp

if (SDst != -1) {
       // VOP1/2 - insert omod if present in instruction
       insertNamedMCOperand(MI, MCOperand::createImm(0), AMDGPU::OpName::omod);
     }
   }
   return MCDisassembler::Success;
 }
 
 DecodeStatus AMDGPUDisassembler::convertMIMGInst(MCInst &MI) const {
+  int VDstIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
+                                           AMDGPU::OpName::vdst);
+
   int VDataIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
                                             AMDGPU::OpName::vdata);
 
   int DMaskIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
                                             AMDGPU::OpName::dmask);
+
+  assert(VDataIdx != -1);
+  assert(DMaskIdx != -1);
+
+  bool isAtomic = (VDstIdx != -1);
+
   unsigned DMask = MI.getOperand(DMaskIdx).getImm() & 0xf;
   if (DMask == 0)
     return MCDisassembler::Success;
 
   unsigned ChannelCount = countPopulation(DMask);
   if (ChannelCount == 1)
     return MCDisassembler::Success;
 
-  int NewOpcode = AMDGPU::getMaskedMIMGOp(*MCII, MI.getOpcode(), ChannelCount);
+  int NewOpcode = -1;
+
+  if (isAtomic) {
+    if (DMask == 0x1 || DMask == 0x3 || DMask == 0xF) {
+      NewOpcode = AMDGPU::getMaskedMIMGAtomicOp(*MCII, MI.getOpcode(), ChannelCount);
+    }
+    if (NewOpcode == -1) return MCDisassembler::Success;
+  } else {
+    NewOpcode = AMDGPU::getMaskedMIMGOp(*MCII, MI.getOpcode(), ChannelCount);
     assert(NewOpcode != -1 && "could not find matching mimg channel instruction");
+  }
+
   auto RCID = MCII->get(NewOpcode).OpInfo[VDataIdx].RegClass;
 
-  // Widen the register to the correct number of enabled channels.
+  // Get first subregister of VData
   unsigned Vdata0 = MI.getOperand(VDataIdx).getReg();
+  unsigned VdataSub0 = MRI.getSubReg(Vdata0, AMDGPU::sub0);
+  Vdata0 = (VdataSub0 != 0)? VdataSub0 : Vdata0;
+
+  // Widen the register to the correct number of enabled channels.
   auto NewVdata = MRI.getMatchingSuperReg(Vdata0, AMDGPU::sub0,
                                           &MRI.getRegClass(RCID));
   if (NewVdata == AMDGPU::NoRegister) {
     // It's possible to encode this such that the low register + enabled
     // components exceeds the register count.
     return MCDisassembler::Success;
   }
 
   MI.setOpcode(NewOpcode);
   // vaddr will be always appear as a single VGPR. This will look different than
   // how it is usually emitted because the number of register components is not
   // in the instruction encoding.
   MI.getOperand(VDataIdx) = MCOperand::createReg(NewVdata);
+
+  if (isAtomic) {
+    // Atomic operations have an additional operand (a copy of data)
+    MI.getOperand(VDstIdx) = MCOperand::createReg(NewVdata);
+  }
+
   return MCDisassembler::Success;
 }
 
 const char* AMDGPUDisassembler::getRegClassName(unsigned RegClassID) const {
   return getContext().getRegisterInfo()->
     getRegClassName(&AMDGPUMCRegisterClasses[RegClassID]);
 }
 

llvm/trunk/lib/Target/AMDGPU/MIMGInstructions.td

 //===-- MIMGInstructions.td - MIMG Instruction Defintions -----------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 class MIMG_Mask <string op, int channels> {
   string Op = op;
   int Channels = channels;
 }
 
+class MIMG_Atomic_Size <string op, bit is32Bit> {
+  string Op = op;
+  int AtomicSize = !if(is32Bit, 1, 2);
+}
+
 class mimg <bits<7> si, bits<7> vi = si> {
   field bits<7> SI = si;
   field bits<7> VI = vi;
 }
 
 class MIMG_Helper <dag outs, dag ins, string asm,
                    string dns=""> : MIMG<outs, ins, asm,[]> {
   let mayLoad = 1;
class MIMG_Atomic_Real_vi<mimg op, string name, string asm,
   MIMG_Atomic_Helper<asm, data_rc, addr_rc, "VI", enableDasm>,
   SIMCInstr<name, SIEncodingFamily.VI>,
   MIMGe<op.VI> {
   let isCodeGenOnly = 0;
   let AssemblerPredicates = [isVI];
   let DisableDecoder = DisableVIDecoder;
 }
 
-multiclass MIMG_Atomic_Helper_m <mimg op, string name, string asm,
+multiclass MIMG_Atomic_Helper_m <mimg op,
+                                 string name,
+                                 string asm,
+                                 string key,
                                  RegisterClass data_rc,
                                  RegisterClass addr_rc,
+                                 bit is32Bit,
                                  bit enableDasm = 0> {
   let isPseudo = 1, isCodeGenOnly = 1 in {
     def "" : MIMG_Atomic_Helper<asm, data_rc, addr_rc>,
              SIMCInstr<name, SIEncodingFamily.NONE>;
   }
 
   let ssamp = 0 in {
-    def _si : MIMG_Atomic_Real_si<op, name, asm, data_rc, addr_rc, enableDasm>;
+    def _si : MIMG_Atomic_Real_si<op, name, asm, data_rc, addr_rc, enableDasm>,
+              MIMG_Atomic_Size<key # "_si", is32Bit>;
 
-    def _vi : MIMG_Atomic_Real_vi<op, name, asm, data_rc, addr_rc, enableDasm>;
+    def _vi : MIMG_Atomic_Real_vi<op, name, asm, data_rc, addr_rc, enableDasm>,
+              MIMG_Atomic_Size<key # "_vi", is32Bit>;
   }
 }
 
-multiclass MIMG_Atomic <mimg op, string asm, RegisterClass data_rc = VGPR_32> {
+multiclass MIMG_Atomic_Addr_Helper_m <mimg op,
+                                      string name,
+                                      string asm,
+                                      RegisterClass data_rc,
+                                      bit is32Bit,
+                                      bit enableDasm = 0> {
   // _V* variants have different address size, but the size is not encoded.
   // So only one variant can be disassembled. V1 looks the safest to decode.
-  defm _V1 : MIMG_Atomic_Helper_m <op, asm # "_V1", asm, data_rc, VGPR_32, 1>;
-  defm _V2 : MIMG_Atomic_Helper_m <op, asm # "_V2", asm, data_rc, VReg_64>;
-  defm _V4 : MIMG_Atomic_Helper_m <op, asm # "_V3", asm, data_rc, VReg_128>;
+  defm _V1 : MIMG_Atomic_Helper_m <op, name # "_V1", asm, asm # "_V1", data_rc, VGPR_32, is32Bit, enableDasm>;
+  defm _V2 : MIMG_Atomic_Helper_m <op, name # "_V2", asm, asm # "_V2", data_rc, VReg_64, is32Bit>;
+  defm _V4 : MIMG_Atomic_Helper_m <op, name # "_V3", asm, asm # "_V3", data_rc, VReg_128, is32Bit>;
+}
+
+multiclass MIMG_Atomic <mimg op, string asm,
+                        RegisterClass data_rc_32 = VGPR_32,   // 32-bit atomics
+                        RegisterClass data_rc_64 = VReg_64> { // 64-bit atomics
+  // _V* variants have different dst size, but the size is encoded implicitly,
+  // using dmask and tfe. Only 32-bit variant is registered with disassembler.
+  // Other variants are reconstructed by disassembler using dmask and tfe.
+  defm _V1 : MIMG_Atomic_Addr_Helper_m <op, asm # "_V1", asm, data_rc_32, 1, 1>;
+  defm _V2 : MIMG_Atomic_Addr_Helper_m <op, asm # "_V2", asm, data_rc_64, 0>;
 }
 
 class MIMG_Sampler_Helper <bits<7> op, string asm,
                            RegisterClass dst_rc,
                            RegisterClass src_rc,
                            bit wqm,
                            bit d16_bit=0,
                            string dns=""> : MIMG_Helper <

 //def IMAGE_STORE_PCK : MIMG_NoPattern_ <"image_store_pck", 0x0000000a>;
 //def IMAGE_STORE_MIP_PCK : MIMG_NoPattern_ <"image_store_mip_pck", 0x0000000b>;
 
 let mayLoad = 0, mayStore = 0 in {
 defm IMAGE_GET_RESINFO : MIMG_NoSampler <0x0000000e, "image_get_resinfo">;
 }
 
 defm IMAGE_ATOMIC_SWAP : MIMG_Atomic <mimg<0x0f, 0x10>, "image_atomic_swap">;
-defm IMAGE_ATOMIC_CMPSWAP : MIMG_Atomic <mimg<0x10, 0x11>, "image_atomic_cmpswap", VReg_64>;
+defm IMAGE_ATOMIC_CMPSWAP : MIMG_Atomic <mimg<0x10, 0x11>, "image_atomic_cmpswap", VReg_64, VReg_128>;
 defm IMAGE_ATOMIC_ADD : MIMG_Atomic <mimg<0x11, 0x12>, "image_atomic_add">;
 defm IMAGE_ATOMIC_SUB : MIMG_Atomic <mimg<0x12, 0x13>, "image_atomic_sub">;
 //def IMAGE_ATOMIC_RSUB : MIMG_NoPattern_ <"image_atomic_rsub", 0x00000013>; -- not on VI
 defm IMAGE_ATOMIC_SMIN : MIMG_Atomic <mimg<0x14>, "image_atomic_smin">;
 defm IMAGE_ATOMIC_UMIN : MIMG_Atomic <mimg<0x15>, "image_atomic_umin">;
 defm IMAGE_ATOMIC_SMAX : MIMG_Atomic <mimg<0x16>, "image_atomic_smax">;
 defm IMAGE_ATOMIC_UMAX : MIMG_Atomic <mimg<0x17>, "image_atomic_umax">;
 defm IMAGE_ATOMIC_AND : MIMG_Atomic <mimg<0x18>, "image_atomic_and">;

 // ImageAtomic for amdgcn.
 class ImageAtomicPattern<SDPatternOperator name, MIMG opcode, ValueType vt> : GCNPat <
   (name i32:$vdata, vt:$addr, v8i32:$rsrc, imm:$r128, imm:$da, imm:$slc),
   (opcode $vdata, $addr, $rsrc, 1, 1, 1, (as_i1imm $slc), (as_i1imm $r128), 0, 0, (as_i1imm $da))
 >;
 
 // ImageAtomic patterns.
 multiclass ImageAtomicPatterns<SDPatternOperator name, string opcode> {
-  def : ImageAtomicPattern<name, !cast<MIMG>(opcode # _V1), i32>;
-  def : ImageAtomicPattern<name, !cast<MIMG>(opcode # _V2), v2i32>;
-  def : ImageAtomicPattern<name, !cast<MIMG>(opcode # _V4), v4i32>;
+  def : ImageAtomicPattern<name, !cast<MIMG>(opcode # _V1_V1), i32>;
+  def : ImageAtomicPattern<name, !cast<MIMG>(opcode # _V1_V2), v2i32>;
+  def : ImageAtomicPattern<name, !cast<MIMG>(opcode # _V1_V4), v4i32>;
 }
 
 // ImageAtomicCmpSwap for amdgcn.
 class ImageAtomicCmpSwapPattern<MIMG opcode, ValueType vt> : GCNPat <
   (int_amdgcn_image_atomic_cmpswap i32:$vsrc, i32:$vcmp, vt:$addr, v8i32:$rsrc,
                                    imm:$r128, imm:$da, imm:$slc),
   (EXTRACT_SUBREG
     (opcode (REG_SEQUENCE VReg_64, $vsrc, sub0, $vcmp, sub1),

 defm : ImageSampleAltPatterns<SIImage_gather4_c_b_o,     "IMAGE_GATHER4_C_B_O">;
 defm : ImageSampleAltPatterns<SIImage_gather4_c_b_cl_o,  "IMAGE_GATHER4_C_B_CL_O">;
 defm : ImageSampleAltPatterns<SIImage_gather4_c_lz_o,    "IMAGE_GATHER4_C_LZ_O">;
 
 defm : ImageSamplePatterns<int_amdgcn_image_getlod, "IMAGE_GET_LOD">;
 
 // Image atomics
 defm : ImageAtomicPatterns<int_amdgcn_image_atomic_swap, "IMAGE_ATOMIC_SWAP">;
-def : ImageAtomicCmpSwapPattern<IMAGE_ATOMIC_CMPSWAP_V1, i32>;
-def : ImageAtomicCmpSwapPattern<IMAGE_ATOMIC_CMPSWAP_V2, v2i32>;
-def : ImageAtomicCmpSwapPattern<IMAGE_ATOMIC_CMPSWAP_V4, v4i32>;
+def : ImageAtomicCmpSwapPattern<IMAGE_ATOMIC_CMPSWAP_V1_V1, i32>;
+def : ImageAtomicCmpSwapPattern<IMAGE_ATOMIC_CMPSWAP_V1_V2, v2i32>;
+def : ImageAtomicCmpSwapPattern<IMAGE_ATOMIC_CMPSWAP_V1_V4, v4i32>;
 defm : ImageAtomicPatterns<int_amdgcn_image_atomic_add, "IMAGE_ATOMIC_ADD">;
 defm : ImageAtomicPatterns<int_amdgcn_image_atomic_sub, "IMAGE_ATOMIC_SUB">;
 defm : ImageAtomicPatterns<int_amdgcn_image_atomic_smin, "IMAGE_ATOMIC_SMIN">;
 defm : ImageAtomicPatterns<int_amdgcn_image_atomic_umin, "IMAGE_ATOMIC_UMIN">;
 defm : ImageAtomicPatterns<int_amdgcn_image_atomic_smax, "IMAGE_ATOMIC_SMAX">;
 defm : ImageAtomicPatterns<int_amdgcn_image_atomic_umax, "IMAGE_ATOMIC_UMAX">;
 defm : ImageAtomicPatterns<int_amdgcn_image_atomic_and, "IMAGE_ATOMIC_AND">;
 defm : ImageAtomicPatterns<int_amdgcn_image_atomic_or, "IMAGE_ATOMIC_OR">;

llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.td

 def getMaskedMIMGOp4 : InstrMapping {
   let FilterClass = "MIMG_Mask";
   let RowFields = ["Op"];
   let ColFields = ["Channels"];
   let KeyCol = ["4"];
   let ValueCols = [["1"], ["2"], ["3"] ];
 }
 
+def getMIMGAtomicOp1 : InstrMapping {
+  let FilterClass = "MIMG_Atomic_Size";
+  let RowFields = ["Op"];
+  let ColFields = ["AtomicSize"];
+  let KeyCol = ["1"];
+  let ValueCols = [["2"]];
+}
+
+def getMIMGAtomicOp2 : InstrMapping {
+  let FilterClass = "MIMG_Atomic_Size";
+  let RowFields = ["Op"];
+  let ColFields = ["AtomicSize"];
+  let KeyCol = ["2"];
+  let ValueCols = [["1"]];
+}
+
 // Maps an commuted opcode to its original version
 def getCommuteOrig : InstrMapping {
   let FilterClass = "Commutable_REV";
   let RowFields = ["RevOp"];
   let ColFields = ["IsOrig"];
   let KeyCol = ["0"];
   let ValueCols = [["1"]];
 }

llvm/trunk/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h

 } // end namespace IsaInfo
 
 LLVM_READONLY
 int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx);
 
 LLVM_READONLY
 int getMaskedMIMGOp(const MCInstrInfo &MII,
                     unsigned Opc, unsigned NewChannels);
+
+LLVM_READONLY
+int getMaskedMIMGAtomicOp(const MCInstrInfo &MII,
+                          unsigned Opc, unsigned NewChannels);
+
 LLVM_READONLY
 int getMCOpcode(uint16_t Opcode, unsigned Gen);
 
 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header,
                                const FeatureBitset &Features);
 
 bool isGroupSegment(const GlobalValue *GV);
 bool isGlobalSegment(const GlobalValue *GV);

llvm/trunk/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

case 3:
     return AMDGPU::getMaskedMIMGOp3(Opc, Channel);
   case 4:
     return AMDGPU::getMaskedMIMGOp4(Opc, Channel);
   default:
     llvm_unreachable("invalid MIMG channel");
   }
 }
 
+int getMaskedMIMGAtomicOp(const MCInstrInfo &MII, unsigned Opc, unsigned NewChannels) {
+  assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst) != -1);
+  assert(NewChannels == 1 || NewChannels == 2 || NewChannels == 4);
+
+  unsigned OrigChannels = rcToChannels(MII.get(Opc).OpInfo[0].RegClass);
+  assert(OrigChannels == 1 || OrigChannels == 2 || OrigChannels == 4);
+
+  if (NewChannels == OrigChannels) return Opc;
+
+  if (OrigChannels <= 2 && NewChannels <= 2) {
+    // This is an ordinary atomic (not an atomic_cmpswap)
+    return (OrigChannels == 1)?
+      AMDGPU::getMIMGAtomicOp1(Opc) : AMDGPU::getMIMGAtomicOp2(Opc);
+  } else if (OrigChannels >= 2 && NewChannels >= 2) {
+    // This is an atomic_cmpswap
+    return (OrigChannels == 2)?
+      AMDGPU::getMIMGAtomicOp1(Opc) : AMDGPU::getMIMGAtomicOp2(Opc);
+  } else { // invalid OrigChannels/NewChannels value
+    return -1;
+  }
+}
+
 // Wrapper for Tablegen'd function.  enum Subtarget is not defined in any
 // header files, so we need to wrap it in a function that takes unsigned
 // instead.
 int getMCOpcode(uint16_t Opcode, unsigned Gen) {
   return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen));
 }
 
 namespace IsaInfo {

llvm/trunk/test/MC/AMDGPU/mimg.s

 //===----------------------------------------------------------------------===//
 // Image Atomics
 //===----------------------------------------------------------------------===//
 
 image_atomic_add v4, v[192:195], s[28:35] dmask:0x1 unorm glc
 // SICI: image_atomic_add v4, v[192:195], s[28:35] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x44,0xf0,0xc0,0x04,0x07,0x00]
 // VI:   image_atomic_add v4, v[192:195], s[28:35] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x48,0xf0,0xc0,0x04,0x07,0x00]
 
-image_atomic_add v5, v1, s[8:15]
-// SICI: image_atomic_add v5, v1, s[8:15] ; encoding: [0x00,0x00,0x44,0xf0,0x01,0x05,0x02,0x00]
-// VI:   image_atomic_add v5, v1, s[8:15] ; encoding: [0x00,0x00,0x48,0xf0,0x01,0x05,0x02,0x00]
-
-image_atomic_add v252, v2, s[8:15] unorm
-// SICI: image_atomic_add v252, v2, s[8:15] unorm ; encoding: [0x00,0x10,0x44,0xf0,0x02,0xfc,0x02,0x00]
-// VI:   image_atomic_add v252, v2, s[8:15] unorm ; encoding: [0x00,0x10,0x48,0xf0,0x02,0xfc,0x02,0x00]
-
-image_atomic_add v6, v255, s[8:15] dmask:0x1
-// SICI: image_atomic_add v6, v255, s[8:15] dmask:0x1 ; encoding: [0x00,0x01,0x44,0xf0,0xff,0x06,0x02,0x00]
-// VI:   image_atomic_add v6, v255, s[8:15] dmask:0x1 ; encoding: [0x00,0x01,0x48,0xf0,0xff,0x06,0x02,0x00]
-
-image_atomic_add v7, v3, s[0:7] glc
-// SICI: image_atomic_add v7, v3, s[0:7] glc ; encoding: [0x00,0x20,0x44,0xf0,0x03,0x07,0x00,0x00]
-// VI:   image_atomic_add v7, v3, s[0:7] glc ; encoding: [0x00,0x20,0x48,0xf0,0x03,0x07,0x00,0x00]
-
-image_atomic_add v8, v4, s[8:15] slc
-// SICI: image_atomic_add v8, v4, s[8:15] slc ; encoding: [0x00,0x00,0x44,0xf2,0x04,0x08,0x02,0x00]
-// VI:   image_atomic_add v8, v4, s[8:15] slc ; encoding: [0x00,0x00,0x48,0xf2,0x04,0x08,0x02,0x00]
+image_atomic_add v252, v2, s[8:15] dmask:0x1 unorm
+// SICI: image_atomic_add v252, v2, s[8:15] dmask:0x1 unorm ; encoding: [0x00,0x11,0x44,0xf0,0x02,0xfc,0x02,0x00]
+// VI:   image_atomic_add v252, v2, s[8:15] dmask:0x1 unorm ; encoding: [0x00,0x11,0x48,0xf0,0x02,0xfc,0x02,0x00]
+
+image_atomic_add v[6:7], v255, s[8:15] dmask:0x3
+// SICI: image_atomic_add v[6:7], v255, s[8:15] dmask:0x3 ; encoding: [0x00,0x03,0x44,0xf0,0xff,0x06,0x02,0x00]
+// VI:   image_atomic_add v[6:7], v255, s[8:15] dmask:0x3 ; encoding: [0x00,0x03,0x48,0xf0,0xff,0x06,0x02,0x00]
+
+image_atomic_add v7, v3, s[0:7] dmask:0x1 glc
+// SICI: image_atomic_add v7, v3, s[0:7] dmask:0x1 glc ; encoding: [0x00,0x21,0x44,0xf0,0x03,0x07,0x00,0x00]
+// VI:   image_atomic_add v7, v3, s[0:7] dmask:0x1 glc ; encoding: [0x00,0x21,0x48,0xf0,0x03,0x07,0x00,0x00]
+
+image_atomic_add v8, v4, s[8:15] dmask:0x1 slc
+// SICI: image_atomic_add v8, v4, s[8:15] dmask:0x1 slc ; encoding: [0x00,0x01,0x44,0xf2,0x04,0x08,0x02,0x00]
+// VI:   image_atomic_add v8, v4, s[8:15] dmask:0x1 slc ; encoding: [0x00,0x01,0x48,0xf2,0x04,0x08,0x02,0x00]
 
 image_atomic_add v9, v5, s[8:15] dmask:0x1 unorm glc slc lwe da
 // SICI: image_atomic_add v9, v5, s[8:15] dmask:0x1 unorm glc slc lwe da ; encoding: [0x00,0x71,0x46,0xf2,0x05,0x09,0x02,0x00]
 // VI:   image_atomic_add v9, v5, s[8:15] dmask:0x1 unorm glc slc lwe da ; encoding: [0x00,0x71,0x4a,0xf2,0x05,0x09,0x02,0x00]
 
 image_atomic_add v10, v6, s[8:15] dmask:0x1 lwe
 // SICI: image_atomic_add v10, v6, s[8:15] dmask:0x1 lwe ; encoding: [0x00,0x01,0x46,0xf0,0x06,0x0a,0x02,0x00]
 // VI:   image_atomic_add v10, v6, s[8:15] dmask:0x1 lwe ; encoding: [0x00,0x01,0x4a,0xf0,0x06,0x0a,0x02,0x00]
 
 image_atomic_add v11, v7, s[8:15] dmask:0x1 da
 // SICI: image_atomic_add v11, v7, s[8:15] dmask:0x1 da ; encoding: [0x00,0x41,0x44,0xf0,0x07,0x0b,0x02,0x00]
 // VI:   image_atomic_add v11, v7, s[8:15] dmask:0x1 da ; encoding: [0x00,0x41,0x48,0xf0,0x07,0x0b,0x02,0x00]
 
 image_atomic_swap v4, v[192:195], s[28:35] dmask:0x1 unorm glc
 // SICI: image_atomic_swap v4, v[192:195], s[28:35] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x3c,0xf0,0xc0,0x04,0x07,0x00]
 // VI:   image_atomic_swap v4, v[192:195], s[28:35] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x40,0xf0,0xc0,0x04,0x07,0x00]
 
-image_atomic_cmpswap v[4:5], v[192:195], s[28:35] dmask:0x1 unorm glc
-// SIIC: image_atomic_cmpswap v[4:5], v[192:195], s[28:35] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x40,0xf0,0xc0,0x04,0x07,0x00]
-// VI:   image_atomic_cmpswap v[4:5], v[192:195], s[28:35] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x44,0xf0,0xc0,0x04,0x07,0x00]
+image_atomic_cmpswap v[4:5], v[192:195], s[28:35] dmask:0x3 unorm glc
+// SICI: image_atomic_cmpswap v[4:5], v[192:195], s[28:35] dmask:0x3 unorm glc ; encoding: [0x00,0x33,0x40,0xf0,0xc0,0x04,0x07,0x00]
+// VI:   image_atomic_cmpswap v[4:5], v[192:195], s[28:35] dmask:0x3 unorm glc ; encoding: [0x00,0x33,0x44,0xf0,0xc0,0x04,0x07,0x00]
+
+image_atomic_cmpswap v[4:7], v[192:195], s[28:35] dmask:0xf unorm glc
+// SICI: image_atomic_cmpswap v[4:7], v[192:195], s[28:35] dmask:0xf unorm glc ; encoding: [0x00,0x3f,0x40,0xf0,0xc0,0x04,0x07,0x00]
+// VI:   image_atomic_cmpswap v[4:7], v[192:195], s[28:35] dmask:0xf unorm glc ; encoding: [0x00,0x3f,0x44,0xf0,0xc0,0x04,0x07,0x00]

llvm/trunk/test/MC/Disassembler/AMDGPU/mimg_vi.txt

 # VI: image_atomic_add v5, v1, s[8:15] dmask:0x1 unorm slc ; encoding: [0x00,0x11,0x48,0xf2,0x01,0x05,0x02,0x00]
 0x00,0x11,0x48,0xf2,0x01,0x05,0x02,0x00
 
 # VI: image_atomic_add v5, v1, s[8:15] dmask:0x1 unorm lwe ; encoding: [0x00,0x11,0x4a,0xf0,0x01,0x05,0x02,0x00]
 0x00,0x11,0x4a,0xf0,0x01,0x05,0x02,0x00
 
 # VI: image_atomic_add v5, v1, s[8:15] dmask:0x1 unorm da ; encoding: [0x00,0x51,0x48,0xf0,0x01,0x05,0x02,0x00]
 0x00,0x51,0x48,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_add v[5:6], v1, s[8:15] dmask:0x3 unorm ; encoding: [0x00,0x13,0x48,0xf0,0x01,0x05,0x02,0x00]
+0x00,0x13,0x48,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_cmpswap v[5:6], v1, s[8:15] dmask:0x3 unorm ; encoding: [0x00,0x13,0x44,0xf0,0x01,0x05,0x02,0x00]
+0x00,0x13,0x44,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_cmpswap v[5:8], v1, s[8:15] dmask:0xf unorm ; encoding: [0x00,0x1f,0x44,0xf0,0x01,0x05,0x02,0x00]
+0x00,0x1f,0x44,0xf0,0x01,0x05,0x02,0x00
+
+#===------------------------------------------------------------------------===#
+# Invalid image atomics (incorrect dmask value).
+# Disassembler may produce a partially incorrect instruction but should not fail.
+#===------------------------------------------------------------------------===#
+
+# VI: image_atomic_add v5, v1, s[8:15] dmask:0x2 unorm ; encoding: [0x00,0x12,0x48,0xf0,0x01,0x05,0x02,0x00]
+0x00,0x12,0x48,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_add v5, v1, s[8:15] dmask:0x7 unorm ; encoding: [0x00,0x17,0x48,0xf0,0x01,0x05,0x02,0x00]
+0x00,0x17,0x48,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_add v5, v1, s[8:15] dmask:0xf unorm ; encoding: [0x00,0x1f,0x48,0xf0,0x01,0x05,0x02,0x00]
+0x00,0x1f,0x48,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_cmpswap v[5:6], v1, s[8:15] unorm ; encoding: [0x00,0x10,0x44,0xf0,0x01,0x05,0x02,0x00]
+0x00,0x10,0x44,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_cmpswap v[5:6], v1, s[8:15] dmask:0x1 unorm ; encoding: [0x00,0x11,0x44,0xf0,0x01,0x05,0x02,0x00]
+0x00,0x11,0x44,0xf0,0x01,0x05,0x02,0x00
+
+# VI: image_atomic_cmpswap v[5:6], v1, s[8:15] dmask:0xe unorm ; encoding: [0x00,0x1e,0x44,0xf0,0x01,0x05,0x02,0x00]
+0x00,0x1e,0x44,0xf0,0x01,0x05,0x02,0x00