[AMDGPU] gfx908 atomic fadd and atomic pk_fadd

[arsenm]

This isn't necessary for the scalar case, as atomicrmw supports fadd

[rampitec]

In D64435#1576720, @arsenm wrote:

This isn't necessary for the scalar case, as atomicrmw supports fadd

Lowering of IR atomics will need to be added separately. These are intrinsics and instructions.

[msearles]

LGTM

llvm/trunk/include/llvm/IR/IntrinsicsAMDGPU.td

[
       llvm_i1_ty   // %clamp
     ],
     [IntrNoMem, IntrSpeculatable, ImmArg<3>]
   >;
 
 //===----------------------------------------------------------------------===//
 // MI-100 intrinsics
 // ===----------------------------------------------------------------------===//
+
+class AMDGPUBufferAtomicNoRtn : Intrinsic <
+  [],
+  [llvm_anyfloat_ty,  // vdata(VGPR)
+   llvm_v4i32_ty,     // rsrc(SGPR)
+   llvm_i32_ty,       // vindex(VGPR)
+   llvm_i32_ty,       // offset(SGPR/VGPR/imm)
+   llvm_i1_ty],       // slc(imm)
+  [], "", [SDNPMemOperand]>,
+  AMDGPURsrcIntrinsic<1, 0>;
+
+class AMDGPUGlobalAtomicNoRtn : Intrinsic <
+  [],
+  [llvm_anyptr_ty,    // vaddr
+   llvm_anyfloat_ty],               // vdata(VGPR)
+  [IntrArgMemOnly, NoCapture<0>], "", [SDNPMemOperand]>;
+
+def int_amdgcn_buffer_atomic_fadd    : AMDGPUBufferAtomicNoRtn;
+def int_amdgcn_global_atomic_fadd    : AMDGPUGlobalAtomicNoRtn;
+
+//===----------------------------------------------------------------------===//
+// MI-100 intrinsics
+// ===----------------------------------------------------------------------===//
 // llvm.amdgcn.mfma.f32.* vdst, srcA, srcB, srcC, cbsz, abid, blgp
 def int_amdgcn_mfma_f32_32x32x1f32 : Intrinsic<[llvm_v32i32_ty],
   [llvm_float_ty, llvm_float_ty, llvm_v32i32_ty,
    llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrConvergent, IntrNoMem]>;
 
 def int_amdgcn_mfma_f32_16x16x1f32 : Intrinsic<[llvm_v16f32_ty],
   [llvm_float_ty, llvm_float_ty, llvm_v16f32_ty,
    llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrConvergent, IntrNoMem]>;

llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.h

enum NodeType : unsigned {
   BUFFER_ATOMIC_SMIN,
   BUFFER_ATOMIC_UMIN,
   BUFFER_ATOMIC_SMAX,
   BUFFER_ATOMIC_UMAX,
   BUFFER_ATOMIC_AND,
   BUFFER_ATOMIC_OR,
   BUFFER_ATOMIC_XOR,
   BUFFER_ATOMIC_CMPSWAP,
+  BUFFER_ATOMIC_FADD,
+  BUFFER_ATOMIC_PK_FADD,
+  ATOMIC_FADD,
+  ATOMIC_PK_FADD,
 
   LAST_AMDGPU_ISD_NUMBER
 };
 
 
 } // End namespace AMDGPUISD
 
 } // End namespace llvm
 
 #endif

llvm/trunk/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(BUFFER_ATOMIC_SMIN)
   NODE_NAME_CASE(BUFFER_ATOMIC_UMIN)
   NODE_NAME_CASE(BUFFER_ATOMIC_SMAX)
   NODE_NAME_CASE(BUFFER_ATOMIC_UMAX)
   NODE_NAME_CASE(BUFFER_ATOMIC_AND)
   NODE_NAME_CASE(BUFFER_ATOMIC_OR)
   NODE_NAME_CASE(BUFFER_ATOMIC_XOR)
   NODE_NAME_CASE(BUFFER_ATOMIC_CMPSWAP)
+  NODE_NAME_CASE(BUFFER_ATOMIC_FADD)
+  NODE_NAME_CASE(BUFFER_ATOMIC_PK_FADD)
+  NODE_NAME_CASE(ATOMIC_FADD)
+  NODE_NAME_CASE(ATOMIC_PK_FADD)
 
   case AMDGPUISD::LAST_AMDGPU_ISD_NUMBER: break;
   }
   return nullptr;
 }
 
 SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand,
                                               SelectionDAG &DAG, int Enabled,

llvm/trunk/lib/Target/AMDGPU/AMDGPUInstructions.td

class AtomicCmpSwapRegion <SDNode cmp_swap_node> : PatFrag<
     (ops node:$ptr, node:$cmp, node:$swap),
     (cmp_swap_node node:$ptr, node:$cmp, node:$swap), [{
       AtomicSDNode *AN = cast<AtomicSDNode>(N);
       return AN->getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
 }]>;
 
 def atomic_cmp_swap_local : AtomicCmpSwapLocal <atomic_cmp_swap>;
 
-multiclass global_binary_atomic_op<SDNode atomic_op> {
-  def "" : PatFrag<
-        (ops node:$ptr, node:$value),
-        (atomic_op node:$ptr, node:$value),
-        [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;}]>;
+class global_binary_atomic_op_frag<SDNode atomic_op> : PatFrag<
 
+multiclass global_binary_atomic_op<SDNode atomic_op> {
+  def "" : global_binary_atomic_op_frag<atomic_op>;
+
   def _noret : PatFrag<
         (ops node:$ptr, node:$value),
         (atomic_op node:$ptr, node:$value),
         [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && (SDValue(N, 0).use_empty());}]>;
 
   def _ret : PatFrag<
         (ops node:$ptr, node:$value),
         (atomic_op node:$ptr, node:$value),

llvm/trunk/lib/Target/AMDGPU/BUFInstructions.td

defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores <
   "buffer_store_format_d16_hi_x", VGPR_32
 >;
 
 } // End HasD16LoadStore
 
 def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1",
                                        int_amdgcn_buffer_wbinvl1>;
 
+let SubtargetPredicate = HasAtomicFaddInsts in {
+
+defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN <
+  "buffer_atomic_add_f32", VGPR_32, f32, atomic_add_global
+>;
+defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN <
+  "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_add_global
+>;
+
+} // End SubtargetPredicate = HasAtomicFaddInsts
+
 //===----------------------------------------------------------------------===//
 // MTBUF Instructions
 //===----------------------------------------------------------------------===//
 
 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_x",     VGPR_32>;
 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xy",    VReg_64>;
 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyz",   VReg_96>;
 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyzw",  VReg_128>;

 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">;
 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">;
 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">;
 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">;
 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">;
 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">;
 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">;
 
+multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt,
+                                       string opcode> {
+  def : GCNPat<
+    (name vt:$vdata_in, v4i32:$rsrc, 0,
+          0, i32:$soffset, imm:$offset,
+          imm:$cachepolicy, 0),
+    (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $vdata_in, $rsrc, $soffset,
+                                        (as_i16imm $offset), (extract_slc $cachepolicy))
+  >;
+
+  def : GCNPat<
+    (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
+          0, i32:$soffset, imm:$offset,
+          imm:$cachepolicy, imm),
+    (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vdata_in, $vindex, $rsrc, $soffset,
+                                       (as_i16imm $offset), (extract_slc $cachepolicy))
+  >;
+
+  def : GCNPat<
+    (name vt:$vdata_in, v4i32:$rsrc, 0,
+          i32:$voffset, i32:$soffset, imm:$offset,
+          imm:$cachepolicy, 0),
+    (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $vdata_in, $voffset, $rsrc, $soffset,
+                                       (as_i16imm $offset), (extract_slc $cachepolicy))
+  >;
+
+  def : GCNPat<
+    (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
+          i32:$voffset, i32:$soffset, imm:$offset,
+          imm:$cachepolicy, imm),
+    (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
+      $vdata_in,
+      (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
+      $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy))
+  >;
+}
+
+defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">;
+defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_pk_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">;
+
 def : GCNPat<
   (SIbuffer_atomic_cmpswap
       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
       0, i32:$soffset, imm:$offset,
       imm:$cachepolicy, 0),
   (EXTRACT_SUBREG
     (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN
       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),

 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_vi <0x6b>;
 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_vi <0x6c>;
 
 def BUFFER_STORE_LDS_DWORD_vi   : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>;
 
 def BUFFER_WBINVL1_vi           : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>;
 def BUFFER_WBINVL1_VOL_vi       : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>;
 
+let SubtargetPredicate = HasAtomicFaddInsts in {
+
+defm BUFFER_ATOMIC_ADD_F32    : MUBUF_Real_AllAddr_vi <0x4d>;
+defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_AllAddr_vi <0x4e>;
+
+} // End SubtargetPredicate = HasAtomicFaddInsts
+
 class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :
   MTBUF_Real<ps>,
   Enc64,
   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
   let AssemblerPredicate = isGFX8GFX9;
   let DecoderNamespace = "GFX8";
 
   let Inst{11-0}  = !if(ps.has_offset, offset, ?);

llvm/trunk/lib/Target/AMDGPU/FLATInstructions.td

let SubtargetPredicate = isGFX10Plus, is_flat_global = 1 in {
   defm GLOBAL_ATOMIC_FCMPSWAP_X2 :
     FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap_x2", VReg_64, f64>;
   defm GLOBAL_ATOMIC_FMIN_X2 :
     FLAT_Global_Atomic_Pseudo<"global_atomic_fmin_x2", VReg_64, f64>;
   defm GLOBAL_ATOMIC_FMAX_X2 :
     FLAT_Global_Atomic_Pseudo<"global_atomic_fmax_x2", VReg_64, f64>;
 } // End SubtargetPredicate = isGFX10Plus, is_flat_global = 1
 
+let SubtargetPredicate = HasAtomicFaddInsts, is_flat_global = 1 in {
+
+defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_NO_RTN <
+  "global_atomic_add_f32", VGPR_32, f32, atomic_add_global
+>;
+defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_NO_RTN <
+  "global_atomic_pk_add_f16", VGPR_32, v2f16, atomic_add_global
+>;
+
+} // End SubtargetPredicate = HasAtomicFaddInsts
 
 //===----------------------------------------------------------------------===//
 // Flat Patterns
 //===----------------------------------------------------------------------===//
 
 // Patterns for global loads with no offset.
 class FlatLoadPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <
   (vt (node (FLATOffset i64:$vaddr, i16:$offset, i1:$slc))),

 >;
 
 class FlatAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt,
                      ValueType data_vt = vt> : GCNPat <
   (vt (node (FLATAtomic i64:$vaddr, i16:$offset, i1:$slc), data_vt:$data)),
   (inst $vaddr, $data, $offset, $slc)
 >;
 
+class FlatAtomicPatNoRtn <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <
+  (node (FLATAtomic i64:$vaddr, i16:$offset, i1:$slc), vt:$data),
+  (inst $vaddr, $data, $offset, $slc)
+>;
+
 class FlatSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt,
                      ValueType data_vt = vt> : GCNPat <
   (vt (node (FLATSignedAtomic i64:$vaddr, i16:$offset, i1:$slc), data_vt:$data)),
   (inst $vaddr, $data, $offset, $slc)
 >;
 
 let OtherPredicates = [HasFlatAddressSpace] in {
 

 def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2i16>;
 def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2f16>;
 def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2i16>;
 def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2f16>;
 }
 
 } // End OtherPredicates = [HasFlatAddressSpace]
 
+def atomic_fadd_global    : global_binary_atomic_op_frag<SIglobal_atomic_fadd>;
+def atomic_pk_fadd_global : global_binary_atomic_op_frag<SIglobal_atomic_pk_fadd>;
+
 let OtherPredicates = [HasFlatGlobalInsts], AddedComplexity = 10 in {
 
 def : FlatLoadSignedPat <GLOBAL_LOAD_UBYTE, az_extloadi8_global, i32>;
 def : FlatLoadSignedPat <GLOBAL_LOAD_SBYTE, sextloadi8_global, i32>;
 def : FlatLoadSignedPat <GLOBAL_LOAD_UBYTE, az_extloadi8_global, i16>;
 def : FlatLoadSignedPat <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>;
 def : FlatLoadSignedPat <GLOBAL_LOAD_USHORT, az_extloadi16_global, i32>;
 def : FlatLoadSignedPat <GLOBAL_LOAD_SSHORT, sextloadi16_global, i32>;

 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_UMAX_X2_RTN, atomic_umax_global, i64>;
 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SMIN_X2_RTN, atomic_min_global, i64>;
 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_UMIN_X2_RTN, atomic_umin_global, i64>;
 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_OR_X2_RTN, atomic_or_global, i64>;
 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SWAP_X2_RTN, atomic_swap_global, i64>;
 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global, i64, v2i64>;
 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_XOR_X2_RTN, atomic_xor_global, i64>;
 
+def : FlatAtomicPatNoRtn <GLOBAL_ATOMIC_ADD_F32,    atomic_fadd_global, f32>;
+def : FlatAtomicPatNoRtn <GLOBAL_ATOMIC_PK_ADD_F16, atomic_pk_fadd_global, v2f16>;
+
 } // End OtherPredicates = [HasFlatGlobalInsts], AddedComplexity = 10
 
 
 //===----------------------------------------------------------------------===//
 // Target
 //===----------------------------------------------------------------------===//
 
 //===----------------------------------------------------------------------===//

 defm SCRATCH_STORE_DWORDX4      : FLAT_Real_AllAddr_gfx10<0x01e>;
 defm SCRATCH_STORE_DWORDX3      : FLAT_Real_AllAddr_gfx10<0x01f>;
 defm SCRATCH_LOAD_UBYTE_D16     : FLAT_Real_AllAddr_gfx10<0x020>;
 defm SCRATCH_LOAD_UBYTE_D16_HI  : FLAT_Real_AllAddr_gfx10<0x021>;
 defm SCRATCH_LOAD_SBYTE_D16     : FLAT_Real_AllAddr_gfx10<0x022>;
 defm SCRATCH_LOAD_SBYTE_D16_HI  : FLAT_Real_AllAddr_gfx10<0x023>;
 defm SCRATCH_LOAD_SHORT_D16     : FLAT_Real_AllAddr_gfx10<0x024>;
 defm SCRATCH_LOAD_SHORT_D16_HI  : FLAT_Real_AllAddr_gfx10<0x025>;
+
+let SubtargetPredicate = HasAtomicFaddInsts in {
+
+defm GLOBAL_ATOMIC_ADD_F32    : FLAT_Real_AllAddr_vi <0x04d>;
+defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Real_AllAddr_vi <0x04e>;
+
+} // End SubtargetPredicate = HasAtomicFaddInsts

llvm/trunk/lib/Target/AMDGPU/SIISelLowering.cpp

case Intrinsic::amdgcn_ds_fmax: {
     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
 
     const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(4));
     if (!Vol->isZero())
       Info.flags |= MachineMemOperand::MOVolatile;
 
     return true;
   }
+  case Intrinsic::amdgcn_buffer_atomic_fadd: {
+    SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+
+    Info.opc = ISD::INTRINSIC_VOID;
+    Info.memVT = MVT::getVT(CI.getOperand(0)->getType());
+    Info.ptrVal = MFI->getBufferPSV(
+      *MF.getSubtarget<GCNSubtarget>().getInstrInfo(),
+      CI.getArgOperand(1));
+    Info.align = 0;
+    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
+
+    const ConstantInt *Vol = dyn_cast<ConstantInt>(CI.getOperand(4));
+    if (!Vol || !Vol->isZero())
+      Info.flags |= MachineMemOperand::MOVolatile;
+
+    return true;
+  }
+  case Intrinsic::amdgcn_global_atomic_fadd: {
+    Info.opc = ISD::INTRINSIC_VOID;
+    Info.memVT = MVT::getVT(CI.getOperand(0)->getType()
+                            ->getPointerElementType());
+    Info.ptrVal = CI.getOperand(0);
+    Info.align = 0;
+    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
+
+    return true;
+  }
   case Intrinsic::amdgcn_ds_append:
   case Intrinsic::amdgcn_ds_consume: {
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     Info.memVT = MVT::getVT(CI.getType());
     Info.ptrVal = CI.getOperand(0);
     Info.align = 0;
     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
 
case Intrinsic::amdgcn_struct_buffer_store_format: {
     EVT VDataType = VData.getValueType().getScalarType();
     if (VDataType == MVT::i8 || VDataType == MVT::i16)
       return handleByteShortBufferStores(DAG, VDataType, DL, Ops, M);
 
     return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(), Ops,
                                    M->getMemoryVT(), M->getMemOperand());
   }
 
+  case Intrinsic::amdgcn_buffer_atomic_fadd: {
+    unsigned Slc = cast<ConstantSDNode>(Op.getOperand(6))->getZExtValue();
+    unsigned IdxEn = 1;
+    if (auto Idx = dyn_cast<ConstantSDNode>(Op.getOperand(4)))
+      IdxEn = Idx->getZExtValue() != 0;
+    SDValue Ops[] = {
+      Chain,
+      Op.getOperand(2), // vdata
+      Op.getOperand(3), // rsrc
+      Op.getOperand(4), // vindex
+      SDValue(),        // voffset -- will be set by setBufferOffsets
+      SDValue(),        // soffset -- will be set by setBufferOffsets
+      SDValue(),        // offset -- will be set by setBufferOffsets
+      DAG.getConstant(Slc << 1, DL, MVT::i32), // cachepolicy
+      DAG.getConstant(IdxEn, DL, MVT::i1), // idxen
+    };
+    setBufferOffsets(Op.getOperand(5), DAG, &Ops[4]);
+    EVT VT = Op.getOperand(2).getValueType();
+
+    auto *M = cast<MemSDNode>(Op);
+    unsigned Opcode = VT.isVector() ? AMDGPUISD::BUFFER_ATOMIC_PK_FADD
+                                    : AMDGPUISD::BUFFER_ATOMIC_FADD;
+
+    return DAG.getMemIntrinsicNode(Opcode, DL, Op->getVTList(), Ops, VT,
+                                   M->getMemOperand());
+  }
+
+  case Intrinsic::amdgcn_global_atomic_fadd: {
+    SDValue Ops[] = {
+      Chain,
+      Op.getOperand(2), // ptr
+      Op.getOperand(3)  // vdata
+    };
+    EVT VT = Op.getOperand(3).getValueType();
+
+    auto *M = cast<MemSDNode>(Op);
+    unsigned Opcode = VT.isVector() ? AMDGPUISD::ATOMIC_PK_FADD
+                                    : AMDGPUISD::ATOMIC_FADD;
+
+    return DAG.getMemIntrinsicNode(Opcode, DL, Op->getVTList(), Ops, VT,
+                                   M->getMemOperand());
+  }
+
   case Intrinsic::amdgcn_end_cf:
     return SDValue(DAG.getMachineNode(AMDGPU::SI_END_CF, DL, MVT::Other,
                                       Op->getOperand(2), Chain), 0);
 
   default: {
     if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
             AMDGPU::getImageDimIntrinsicInfo(IntrinsicID))
       return lowerImage(Op, ImageDimIntr, DAG);

llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.td

SDTypeProfile<1, 8,
        SDTCisVT<4, i32>,   // voffset(VGPR)
        SDTCisVT<5, i32>,   // soffset(SGPR)
        SDTCisVT<6, i32>,   // offset(imm)
        SDTCisVT<7, i32>,   // cachepolicy(imm)
        SDTCisVT<8, i1>]>,  // idxen(imm)
   [SDNPMemOperand, SDNPHasChain, SDNPMayLoad, SDNPMayStore]
 >;
 
+class SDBufferAtomicNoRtn<string opcode, ValueType ty> : SDNode <opcode,
+  SDTypeProfile<0, 8,
+      [SDTCisVT<0, ty>,    // vdata
+       SDTCisVT<1, v4i32>, // rsrc
+       SDTCisVT<2, i32>,   // vindex(VGPR)
+       SDTCisVT<3, i32>,   // voffset(VGPR)
+       SDTCisVT<4, i32>,   // soffset(SGPR)
+       SDTCisVT<5, i32>,   // offset(imm)
+       SDTCisVT<6, i32>,   // cachepolicy(imm)
+       SDTCisVT<7, i1>]>,  // idxen(imm)
+  [SDNPMemOperand, SDNPHasChain, SDNPMayLoad, SDNPMayStore]
+>;
+
 def SIbuffer_atomic_swap : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SWAP">;
 def SIbuffer_atomic_add : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_ADD">;
 def SIbuffer_atomic_sub : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SUB">;
 def SIbuffer_atomic_smin : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SMIN">;
 def SIbuffer_atomic_umin : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_UMIN">;
 def SIbuffer_atomic_smax : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SMAX">;
 def SIbuffer_atomic_umax : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_UMAX">;
 def SIbuffer_atomic_and : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_AND">;
 def SIbuffer_atomic_or : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_OR">;
 def SIbuffer_atomic_xor : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_XOR">;
+def SIbuffer_atomic_fadd : SDBufferAtomicNoRtn <"AMDGPUISD::BUFFER_ATOMIC_FADD", f32>;
+def SIbuffer_atomic_pk_fadd : SDBufferAtomicNoRtn <"AMDGPUISD::BUFFER_ATOMIC_PK_FADD", v2f16>;
 
 def SIbuffer_atomic_cmpswap : SDNode <"AMDGPUISD::BUFFER_ATOMIC_CMPSWAP",
   SDTypeProfile<1, 9,
     [SDTCisVT<0, i32>,   // dst
      SDTCisVT<1, i32>,   // src
      SDTCisVT<2, i32>,   // cmp
      SDTCisVT<3, v4i32>, // rsrc
      SDTCisVT<4, i32>,   // vindex(VGPR)
      SDTCisVT<5, i32>,   // voffset(VGPR)
      SDTCisVT<6, i32>,   // soffset(SGPR)
      SDTCisVT<7, i32>,   // offset(imm)
      SDTCisVT<8, i32>,   // cachepolicy(imm)
      SDTCisVT<9, i1>]>,  // idxen(imm)
   [SDNPMemOperand, SDNPHasChain, SDNPMayLoad, SDNPMayStore]
 >;
 
+class SDGlobalAtomicNoRtn<string opcode, ValueType ty> : SDNode <opcode,
+  SDTypeProfile<0, 2,
+      [SDTCisPtrTy<0>,     // vaddr
+       SDTCisVT<1, ty>]>,  // vdata
+  [SDNPMemOperand, SDNPHasChain, SDNPMayLoad, SDNPMayStore]
+>;
+
+def SIglobal_atomic_fadd    : SDGlobalAtomicNoRtn <"AMDGPUISD::ATOMIC_FADD", f32>;
+def SIglobal_atomic_pk_fadd : SDGlobalAtomicNoRtn <"AMDGPUISD::ATOMIC_PK_FADD", v2f16>;
+
 def SIpc_add_rel_offset : SDNode<"AMDGPUISD::PC_ADD_REL_OFFSET",
   SDTypeProfile<1, 2, [SDTCisVT<0, iPTR>, SDTCisSameAs<0,1>, SDTCisSameAs<0,2>]>
 >;
 
 def SIlds : SDNode<"AMDGPUISD::LDS",
   SDTypeProfile<1, 1, [SDTCisVT<0, iPTR>, SDTCisSameAs<0,1>]>
 >;
 

llvm/trunk/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.fadd.ll

+; RUN: llc < %s -march=amdgcn -mcpu=gfx908 -verify-machineinstrs | FileCheck %s -check-prefix=GCN
+
+declare void @llvm.amdgcn.buffer.atomic.fadd.f32(float, <4 x i32>, i32, i32, i1)
+declare void @llvm.amdgcn.buffer.atomic.fadd.v2f16(<2 x half>, <4 x i32>, i32, i32, i1)
+declare void @llvm.amdgcn.global.atomic.fadd.p1f32.f32(float addrspace(1)*, float)
+declare void @llvm.amdgcn.global.atomic.fadd.p1v2f16.v2f16(<2 x half> addrspace(1)*, <2 x half>)
+
+; GCN-LABEL: {{^}}buffer_atomic_add_f32:
+; GCN: buffer_atomic_add_f32 v0, v1, s[0:3], 0 idxen
+define amdgpu_ps void @buffer_atomic_add_f32(<4 x i32> inreg %rsrc, float %data, i32 %vindex) {
+main_body:
+  call void @llvm.amdgcn.buffer.atomic.fadd.f32(float %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
+  ret void
+}
+
+; GCN-LABEL: {{^}}buffer_atomic_add_f32_off4_slc:
+; GCN: buffer_atomic_add_f32 v0, v1, s[0:3], 0 idxen offset:4 slc
+define amdgpu_ps void @buffer_atomic_add_f32_off4_slc(<4 x i32> inreg %rsrc, float %data, i32 %vindex) {
+main_body:
+  call void @llvm.amdgcn.buffer.atomic.fadd.f32(float %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i1 1)
+  ret void
+}
+
+; GCN-LABEL: {{^}}buffer_atomic_pk_add_v2f16:
+; GCN: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 idxen
+define amdgpu_ps void @buffer_atomic_pk_add_v2f16(<4 x i32> inreg %rsrc, <2 x half> %data, i32 %vindex) {
+main_body:
+  call void @llvm.amdgcn.buffer.atomic.fadd.v2f16(<2 x half> %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
+  ret void
+}
+
+; GCN-LABEL: {{^}}buffer_atomic_pk_add_v2f16_off4_slc:
+; GCN: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 idxen offset:4 slc
+define amdgpu_ps void @buffer_atomic_pk_add_v2f16_off4_slc(<4 x i32> inreg %rsrc, <2 x half> %data, i32 %vindex) {
+main_body:
+  call void @llvm.amdgcn.buffer.atomic.fadd.v2f16(<2 x half> %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i1 1)
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_atomic_add_f32:
+; GCN: global_atomic_add_f32 v[{{[0-9:]+}}], v{{[0-9]+}}, off
+define amdgpu_kernel void @global_atomic_add_f32(float addrspace(1)* %ptr, float %data) {
+main_body:
+  call void @llvm.amdgcn.global.atomic.fadd.p1f32.f32(float addrspace(1)* %ptr, float %data)
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_atomic_add_f32_off4:
+; GCN: global_atomic_add_f32 v[{{[0-9:]+}}], v{{[0-9]+}}, off offset:4
+define amdgpu_kernel void @global_atomic_add_f32_off4(float addrspace(1)* %ptr, float %data) {
+main_body:
+  %p = getelementptr float, float addrspace(1)* %ptr, i64 1
+  call void @llvm.amdgcn.global.atomic.fadd.p1f32.f32(float addrspace(1)* %p, float %data)
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_atomic_pk_add_v2f16:
+; GCN: global_atomic_pk_add_f16 v[{{[0-9:]+}}], v{{[0-9]+}}, off
+define amdgpu_kernel void @global_atomic_pk_add_v2f16(<2 x half> addrspace(1)* %ptr, <2 x half> %data) {
+main_body:
+  call void @llvm.amdgcn.global.atomic.fadd.p1v2f16.v2f16(<2 x half> addrspace(1)* %ptr, <2 x half> %data)
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_atomic_pk_add_v2f16_off4:
+; GCN: global_atomic_pk_add_f16 v[{{[0-9:]+}}], v{{[0-9]+}}, off offset:4
+define amdgpu_kernel void @global_atomic_pk_add_v2f16_off4(<2 x half> addrspace(1)* %ptr, <2 x half> %data) {
+main_body:
+  %p = getelementptr <2 x half>, <2 x half> addrspace(1)* %ptr, i64 1
+  call void @llvm.amdgcn.global.atomic.fadd.p1v2f16.v2f16(<2 x half> addrspace(1)* %p, <2 x half> %data)
+  ret void
+}

llvm/trunk/test/MC/AMDGPU/atomic-fadd-insts.s

+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx908 -show-encoding %s | FileCheck --check-prefix=GFX908 %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx908 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX908-ERR %s
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0x03]
+
+buffer_atomic_add_f32 v255, off, s[8:11], s3 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x34,0xe1,0x00,0xff,0x02,0x03]
+
+buffer_atomic_add_f32 v5, off, s[12:15], s3 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x34,0xe1,0x00,0x05,0x03,0x03]
+
+buffer_atomic_add_f32 v5, off, s[96:99], s3 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x34,0xe1,0x00,0x05,0x18,0x03]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s101 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0x65]
+
+buffer_atomic_add_f32 v5, off, s[8:11], m0 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0x7c]
+
+buffer_atomic_add_f32 v5, off, s[8:11], 0 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0x80]
+
+buffer_atomic_add_f32 v5, off, s[8:11], -1 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0xc1]
+
+buffer_atomic_add_f32 v5, v0, s[8:11], s3 idxen offset:4095
+// GFX908: encoding: [0xff,0x2f,0x34,0xe1,0x00,0x05,0x02,0x03]
+
+buffer_atomic_add_f32 v5, v0, s[8:11], s3 offen offset:4095
+// GFX908: encoding: [0xff,0x1f,0x34,0xe1,0x00,0x05,0x02,0x03]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3
+// GFX908: encoding: [0x00,0x00,0x34,0xe1,0x00,0x05,0x02,0x03]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3
+// GFX908: encoding: [0x00,0x00,0x34,0xe1,0x00,0x05,0x02,0x03]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:7
+// GFX908: encoding: [0x07,0x00,0x34,0xe1,0x00,0x05,0x02,0x03]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:4095 glc
+// GFX908-ERR: error: invalid operand for instruction
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:4095 slc
+// GFX908: encoding: [0xff,0x0f,0x36,0xe1,0x00,0x05,0x02,0x03]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0x03]
+
+buffer_atomic_pk_add_f16 v255, off, s[8:11], s3 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x38,0xe1,0x00,0xff,0x02,0x03]
+
+buffer_atomic_pk_add_f16 v5, off, s[12:15], s3 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x38,0xe1,0x00,0x05,0x03,0x03]
+
+buffer_atomic_pk_add_f16 v5, off, s[96:99], s3 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x38,0xe1,0x00,0x05,0x18,0x03]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s101 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0x65]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], m0 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0x7c]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], 0 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0x80]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], -1 offset:4095
+// GFX908: encoding: [0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0xc1]
+
+buffer_atomic_pk_add_f16 v5, v0, s[8:11], s3 idxen offset:4095
+// GFX908: encoding: [0xff,0x2f,0x38,0xe1,0x00,0x05,0x02,0x03]
+
+buffer_atomic_pk_add_f16 v5, v0, s[8:11], s3 offen offset:4095
+// GFX908: encoding: [0xff,0x1f,0x38,0xe1,0x00,0x05,0x02,0x03]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s3
+// GFX908: encoding: [0x00,0x00,0x38,0xe1,0x00,0x05,0x02,0x03]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s3
+// GFX908: encoding: [0x00,0x00,0x38,0xe1,0x00,0x05,0x02,0x03]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:7
+// GFX908: encoding: [0x07,0x00,0x38,0xe1,0x00,0x05,0x02,0x03]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:4095 glc
+// GFX908-ERR: error: invalid operand for instruction
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:4095 slc
+// GFX908: encoding: [0xff,0x0f,0x3a,0xe1,0x00,0x05,0x02,0x03]
+
+global_atomic_add_f32 v[1:2], v2, off offset:-1
+// GFX908: encoding: [0xff,0x9f,0x34,0xdd,0x01,0x02,0x7f,0x00]
+
+global_atomic_add_f32 v[1:2], v255, off offset:-1
+// GFX908: encoding: [0xff,0x9f,0x34,0xdd,0x01,0xff,0x7f,0x00]
+
+global_atomic_add_f32 v[1:2], v2, off
+// GFX908: encoding: [0x00,0x80,0x34,0xdd,0x01,0x02,0x7f,0x00]
+
+global_atomic_pk_add_f16 v[1:2], v2, off offset:-1
+// GFX908: encoding: [0xff,0x9f,0x38,0xdd,0x01,0x02,0x7f,0x00]
+
+global_atomic_pk_add_f16 v[1:2], v255, off offset:-1
+// GFX908: encoding: [0xff,0x9f,0x38,0xdd,0x01,0xff,0x7f,0x00]
+
+global_atomic_pk_add_f16 v[1:2], v2, off
+// GFX908: encoding: [0x00,0x80,0x38,0xdd,0x01,0x02,0x7f,0x00]

llvm/trunk/test/MC/Disassembler/AMDGPU/atomic-fadd-insts.txt

+# RUN: llvm-mc -arch=amdgcn -mcpu=gfx908 -disassemble -show-encoding < %s | FileCheck %s
+
+# CHECK: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:4095 ; encoding: [0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0x03]
+0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_add_f32 v255, off, s[8:11], s3 offset:4095 ; encoding: [0xff,0x0f,0x34,0xe1,0x00,0xff,0x02,0x03]
+0xff,0x0f,0x34,0xe1,0x00,0xff,0x02,0x03
+
+# CHECK: buffer_atomic_add_f32 v5, off, s[12:15], s3 offset:4095 ; encoding: [0xff,0x0f,0x34,0xe1,0x00,0x05,0x03,0x03]
+0xff,0x0f,0x34,0xe1,0x00,0x05,0x03,0x03
+
+# CHECK: buffer_atomic_add_f32 v5, off, s[96:99], s3 offset:4095 ; encoding: [0xff,0x0f,0x34,0xe1,0x00,0x05,0x18,0x03]
+0xff,0x0f,0x34,0xe1,0x00,0x05,0x18,0x03
+
+# CHECK: buffer_atomic_add_f32 v5, off, s[8:11], s101 offset:4095 ; encoding: [0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0x65]
+0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0x65
+
+# CHECK: buffer_atomic_add_f32 v5, off, s[8:11], m0 offset:4095 ; encoding: [0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0x7c]
+0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0x7c
+
+# CHECK: buffer_atomic_add_f32 v5, off, s[8:11], 0 offset:4095 ; encoding: [0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0x80]
+0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0x80
+
+# CHECK: buffer_atomic_add_f32 v5, off, s[8:11], -1 offset:4095 ; encoding: [0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0xc1]
+0xff,0x0f,0x34,0xe1,0x00,0x05,0x02,0xc1
+
+# CHECK: buffer_atomic_add_f32 v5, v0, s[8:11], s3 idxen offset:4095 ; encoding: [0xff,0x2f,0x34,0xe1,0x00,0x05,0x02,0x03]
+0xff,0x2f,0x34,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_add_f32 v5, v0, s[8:11], s3 offen offset:4095 ; encoding: [0xff,0x1f,0x34,0xe1,0x00,0x05,0x02,0x03]
+0xff,0x1f,0x34,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_add_f32 v5, off, s[8:11], s3 ; encoding: [0x00,0x00,0x34,0xe1,0x00,0x05,0x02,0x03]
+0x00,0x00,0x34,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_add_f32 v5, off, s[8:11], s3 ; encoding: [0x00,0x00,0x34,0xe1,0x00,0x05,0x02,0x03]
+0x00,0x00,0x34,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x07,0x00,0x34,0xe1,0x00,0x05,0x02,0x03]
+0x07,0x00,0x34,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:4095 slc ; encoding: [0xff,0x0f,0x36,0xe1,0x00,0x05,0x02,0x03]
+0xff,0x0f,0x36,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:4095 ; encoding: [0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0x03]
+0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_pk_add_f16 v255, off, s[8:11], s3 offset:4095 ; encoding: [0xff,0x0f,0x38,0xe1,0x00,0xff,0x02,0x03]
+0xff,0x0f,0x38,0xe1,0x00,0xff,0x02,0x03
+
+# CHECK: buffer_atomic_pk_add_f16 v5, off, s[12:15], s3 offset:4095 ; encoding: [0xff,0x0f,0x38,0xe1,0x00,0x05,0x03,0x03]
+0xff,0x0f,0x38,0xe1,0x00,0x05,0x03,0x03
+
+# CHECK: buffer_atomic_pk_add_f16 v5, off, s[96:99], s3 offset:4095 ; encoding: [0xff,0x0f,0x38,0xe1,0x00,0x05,0x18,0x03]
+0xff,0x0f,0x38,0xe1,0x00,0x05,0x18,0x03
+
+# CHECK: buffer_atomic_pk_add_f16 v5, off, s[8:11], s101 offset:4095 ; encoding: [0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0x65]
+0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0x65
+
+# CHECK: buffer_atomic_pk_add_f16 v5, off, s[8:11], m0 offset:4095 ; encoding: [0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0x7c]
+0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0x7c
+
+# CHECK: buffer_atomic_pk_add_f16 v5, off, s[8:11], 0 offset:4095 ; encoding: [0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0x80]
+0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0x80
+
+# CHECK: buffer_atomic_pk_add_f16 v5, off, s[8:11], -1 offset:4095 ; encoding: [0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0xc1]
+0xff,0x0f,0x38,0xe1,0x00,0x05,0x02,0xc1
+
+# CHECK: buffer_atomic_pk_add_f16 v5, v0, s[8:11], s3 idxen offset:4095 ; encoding: [0xff,0x2f,0x38,0xe1,0x00,0x05,0x02,0x03]
+0xff,0x2f,0x38,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_pk_add_f16 v5, v0, s[8:11], s3 offen offset:4095 ; encoding: [0xff,0x1f,0x38,0xe1,0x00,0x05,0x02,0x03]
+0xff,0x1f,0x38,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 ; encoding: [0x00,0x00,0x38,0xe1,0x00,0x05,0x02,0x03]
+0x00,0x00,0x38,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 ; encoding: [0x00,0x00,0x38,0xe1,0x00,0x05,0x02,0x03]
+0x00,0x00,0x38,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:7 ; encoding: [0x07,0x00,0x38,0xe1,0x00,0x05,0x02,0x03]
+0x07,0x00,0x38,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:4095 slc ; encoding: [0xff,0x0f,0x3a,0xe1,0x00,0x05,0x02,0x03]
+0xff,0x0f,0x3a,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: global_atomic_add_f32 v[1:2], v2, off offset:-1 ; encoding: [0xff,0x9f,0x34,0xdd,0x01,0x02,0x7f,0x00]
+0xff,0x9f,0x34,0xdd,0x01,0x02,0x7f,0x00
+
+# CHECK: global_atomic_add_f32 v[1:2], v255, off offset:-1 ; encoding: [0xff,0x9f,0x34,0xdd,0x01,0xff,0x7f,0x00]
+0xff,0x9f,0x34,0xdd,0x01,0xff,0x7f,0x00
+
+# CHECK: global_atomic_add_f32 v[1:2], v2, off ; encoding: [0x00,0x80,0x34,0xdd,0x01,0x02,0x7f,0x00]
+0x00,0x80,0x34,0xdd,0x01,0x02,0x7f,0x00
+
+# CHECK: global_atomic_pk_add_f16 v[1:2], v2, off offset:-1 ; encoding: [0xff,0x9f,0x38,0xdd,0x01,0x02,0x7f,0x00]
+0xff,0x9f,0x38,0xdd,0x01,0x02,0x7f,0x00
+
+# CHECK: global_atomic_pk_add_f16 v[1:2], v255, off offset:-1 ; encoding: [0xff,0x9f,0x38,0xdd,0x01,0xff,0x7f,0x00]
+0xff,0x9f,0x38,0xdd,0x01,0xff,0x7f,0x00
+
+# CHECK: global_atomic_pk_add_f16 v[1:2], v2, off ; encoding: [0x00,0x80,0x38,0xdd,0x01,0x02,0x7f,0x00]
+0x00,0x80,0x38,0xdd,0x01,0x02,0x7f,0x00