[AMDGPU] Enable `s_sendmsg_rtn` selection with `+gfx11-insts`

[Pierre-vh]

Summary

ROCm device libs can emit a s_sendmsg_rtn w/ the +gfx11-insts attribute, and it counts on the optimizer to remove the call if the GPU is <GFX11.
When built at O0 it caused codegen issues as Clang allowed this intrinsic to go through with just +gfx11-insts, but the backend wanted the GPU to be >=GFX11 as well.

This patch allows selecting that intrinsic when just the +gfx11-insts attribute is present.

[Pierre-vh]

Fix lint

[foad]

I don't really like the fact that we are left with both isGFX11Plus and HasGFX11Insts, and no clear way to know which one to use, unless you happen to know what instructions some ROCm libraries are relying on.

Seems reasonable apart from that.

[foad]

I do think this is much nicer than the previous approach of generating V_ILLEGAL instead of diagnosing illegal intrinsics (D123693).

[Pierre-vh]

The goal eventually is to move all instructions to just relying on attribute(s) instead of GPU generations I believe. However, that's a lot of work with a lot of risks of breaking things so we agreed with @arsen to just do the necessary work to fix Device Libs for now

[arsenm]

In D136944#3891713, @foad wrote:

I do think this is much nicer than the previous approach of generating V_ILLEGAL instead of diagnosing illegal intrinsics (D123693).

We can't really avoid the V_ILLEGAL case if the instruction isn't encodable for the end subtarget. There's 3 different ways instructions can be illegal

[arsenm]

In D136944#3891707, @foad wrote:

I don't really like the fact that we are left with both isGFX11Plus and HasGFX11Insts, and no clear way to know which one to use, unless you happen to know what instructions some ROCm libraries are relying on.

Seems reasonable apart from that.

Generation checks should always be "wrong". Ideally the hardware would move in nice, modular feature based changes but that's not what has happened

[arsenm]

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

8002–8004

I'm surprised this is necessary. How was this missing before? I thought the encoding didn't change in 11?

[foad]

In D136944#3898249, @arsenm wrote:

In D136944#3891713, @foad wrote:

I do think this is much nicer than the previous approach of generating V_ILLEGAL instead of diagnosing illegal intrinsics (D123693).

We can't really avoid the V_ILLEGAL case if the instruction isn't encodable for the end subtarget.

Doesn't this patch solve that by adding stuff to pseudoToMCOpcode to encode it for GFX11 if it can't find an encoding for the real subtarget?

There's 3 different ways instructions can be illegal

What are they?

[arsenm]

In D136944#3899931, @foad wrote:

There's 3 different ways instructions can be illegal

What are they?

1. Instruction doesn't exist for the subtarget, but is encodable. For example, all instructions added in CI aren't available for SI, but are encodable since nothing changed there.
2. Instruction does not exist for the subtarget, but the opcode was recycled into something else. This is the case for when v_mad_mix* was replaced with v_fma_mix* between gfx900 and gfx906
3. The instruction doesn't exist on the subtarget, and also isn't encodable. This is what happened with the image instructions for MI-200

[Pierre-vh]

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

8002–8004

Without it, I run into `Opcode < NumOpcodes && "Invalid opcode!" without it because that function ends up returning -1

llvm/lib/Target/AMDGPU/AMDGPU.td

 def isGFX11Only :
   Predicate<"Subtarget->getGeneration() == AMDGPUSubtarget::GFX11">,
   AssemblerPredicate<(all_of FeatureGFX11Insts)>;
 
 def isGFX11Plus :
   Predicate<"Subtarget->getGeneration() >= AMDGPUSubtarget::GFX11">,
   AssemblerPredicate<(all_of FeatureGFX11Insts)>;
 
+def HasGFX11Insts :
+  Predicate<"Subtarget->hasGFX11Insts()">,
+  AssemblerPredicate<(all_of FeatureGFX11Insts)>;
+
 def HasFlatAddressSpace : Predicate<"Subtarget->hasFlatAddressSpace()">,
   AssemblerPredicate<(all_of FeatureFlatAddressSpace)>;
 
 def HasFlatGlobalInsts : Predicate<"Subtarget->hasFlatGlobalInsts()">,
   AssemblerPredicate<(all_of FeatureFlatGlobalInsts)>;
 def HasFlatScratchInsts : Predicate<"Subtarget->hasFlatScratchInsts()">,
   AssemblerPredicate<(all_of FeatureFlatScratchInsts)>;
 def HasScalarFlatScratchInsts : Predicate<"Subtarget->hasScalarFlatScratchInsts()">,

llvm/lib/Target/AMDGPU/GCNSubtarget.h

public:
   bool hasLDSFPAtomicAdd() const { return GFX8Insts; }
 
   /// \returns true if the subtarget has the v_permlanex16_b32 instruction.
   bool hasPermLaneX16() const { return getGeneration() >= GFX10; }
 
   /// \returns true if the subtarget has the v_permlane64_b32 instruction.
   bool hasPermLane64() const { return getGeneration() >= GFX11; }
 
+  bool hasGFX11Insts() const { return GFX11Insts; }
+
   bool hasDPP() const {
     return HasDPP;
   }
 
   bool hasDPPBroadcasts() const {
     return HasDPP && getGeneration() < GFX10;
   }
 

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

if (ST.hasGFX90AInsts()) {
     if (NMCOp == (uint16_t)-1)
       NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX90A);
     if (NMCOp == (uint16_t)-1)
       NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX9);
     if (NMCOp != (uint16_t)-1)
       MCOp = NMCOp;
   }
 
+  if (ST.hasGFX11Insts()) {
+    uint16_t NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX11);
+    if (NMCOp != (uint16_t)-1)
+      MCOp = NMCOp;
+  }
+
   // (uint16_t)-1 means that Opcode is a pseudo instruction that has
   // no encoding in the given subtarget generation.
   if (MCOp == (uint16_t)-1)
     return -1;
 
   if (isAsmOnlyOpcode(MCOp))
     return -1;
 

llvm/lib/Target/AMDGPU/SOPInstructions.td

let hasSideEffects = 1, Defs = [EXEC, SCC], Uses = [EXEC] in {
     def S_ANDN2_WREXEC_B32   : SOP1_32<"s_andn2_wrexec_b32">;
   } // End hasSideEffects = 1, Defs = [EXEC, SCC], Uses = [EXEC]
 
   let Uses = [M0] in {
     def S_MOVRELSD_2_B32 : SOP1_32<"s_movrelsd_2_b32">;
   } // End Uses = [M0]
 } // End SubtargetPredicate = isGFX10Plus
 
-let SubtargetPredicate = isGFX11Plus in {
+let SubtargetPredicate = HasGFX11Insts in {
   let hasSideEffects = 1 in {
     // For s_sendmsg_rtn_* the src0 field encodes the message type directly; it
     // is not an SGPR number.
     def S_SENDMSG_RTN_B32 : SOP1_Pseudo<
       "s_sendmsg_rtn_b32", (outs SReg_32:$sdst), (ins SendMsgImm:$src0),
       "$sdst, $src0", [(set i32:$sdst, (int_amdgcn_s_sendmsg_rtn timm:$src0))]
     >;
     def S_SENDMSG_RTN_B64 : SOP1_Pseudo<

llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp

 
 } // namespace DepCtr
 
 namespace SendMsg {
 
 // Disable lint checking for this block since it makes the table unreadable.
 // NOLINTBEGIN
 const CustomOperand<const MCSubtargetInfo &> Msg[] = {
     {{""}},
-  {{"MSG_INTERRUPT"},           ID_INTERRUPT},
-  {{"MSG_GS"},                  ID_GS_PreGFX11,             isNotGFX11Plus},
-  {{"MSG_GS_DONE"},             ID_GS_DONE_PreGFX11,        isNotGFX11Plus},
-  {{"MSG_SAVEWAVE"},            ID_SAVEWAVE,                isGFX8_GFX9_GFX10},
-  {{"MSG_STALL_WAVE_GEN"},      ID_STALL_WAVE_GEN,          isGFX9Plus},
-  {{"MSG_HALT_WAVES"},          ID_HALT_WAVES,              isGFX9Plus},
-  {{"MSG_ORDERED_PS_DONE"},     ID_ORDERED_PS_DONE,         isGFX9Plus},
-  {{"MSG_EARLY_PRIM_DEALLOC"},  ID_EARLY_PRIM_DEALLOC,      isGFX9_GFX10},
-  {{"MSG_GS_ALLOC_REQ"},        ID_GS_ALLOC_REQ,            isGFX9Plus},
-  {{"MSG_GET_DOORBELL"},        ID_GET_DOORBELL,            isGFX9_GFX10},
-  {{"MSG_GET_DDID"},            ID_GET_DDID,                isGFX10},
-  {{"MSG_HS_TESSFACTOR"},       ID_HS_TESSFACTOR_GFX11Plus, isGFX11Plus},
-  {{"MSG_DEALLOC_VGPRS"},       ID_DEALLOC_VGPRS_GFX11Plus, isGFX11Plus},
+    {{"MSG_INTERRUPT"}, ID_INTERRUPT},
+    {{"MSG_GS"}, ID_GS_PreGFX11, isNotGFX11Plus},
+    {{"MSG_GS_DONE"}, ID_GS_DONE_PreGFX11, isNotGFX11Plus},
+    {{"MSG_SAVEWAVE"}, ID_SAVEWAVE, isGFX8_GFX9_GFX10},
+    {{"MSG_STALL_WAVE_GEN"}, ID_STALL_WAVE_GEN, isGFX9Plus},
+    {{"MSG_HALT_WAVES"}, ID_HALT_WAVES, isGFX9Plus},
+    {{"MSG_ORDERED_PS_DONE"}, ID_ORDERED_PS_DONE, isGFX9Plus},
+    {{"MSG_EARLY_PRIM_DEALLOC"}, ID_EARLY_PRIM_DEALLOC, isGFX9_GFX10},
+    {{"MSG_GS_ALLOC_REQ"}, ID_GS_ALLOC_REQ, isGFX9Plus},
+    {{"MSG_GET_DOORBELL"}, ID_GET_DOORBELL, isGFX9_GFX10},
+    {{"MSG_GET_DDID"}, ID_GET_DDID, isGFX10},
+    {{"MSG_HS_TESSFACTOR"}, ID_HS_TESSFACTOR_GFX11Plus, isGFX11Plus},
+    {{"MSG_DEALLOC_VGPRS"}, ID_DEALLOC_VGPRS_GFX11Plus, isGFX11Plus},
     {{""}},
-  {{"MSG_SYSMSG"},              ID_SYSMSG},
-  {{"MSG_RTN_GET_DOORBELL"},    ID_RTN_GET_DOORBELL,        isGFX11Plus},
-  {{"MSG_RTN_GET_DDID"},        ID_RTN_GET_DDID,            isGFX11Plus},
-  {{"MSG_RTN_GET_TMA"},         ID_RTN_GET_TMA,             isGFX11Plus},
-  {{"MSG_RTN_GET_REALTIME"},    ID_RTN_GET_REALTIME,        isGFX11Plus},
-  {{"MSG_RTN_SAVE_WAVE"},       ID_RTN_SAVE_WAVE,           isGFX11Plus},
-  {{"MSG_RTN_GET_TBA"},         ID_RTN_GET_TBA,             isGFX11Plus},
+    {{"MSG_SYSMSG"}, ID_SYSMSG},
+    {{"MSG_RTN_GET_DOORBELL"}, ID_RTN_GET_DOORBELL, hasGFX11Insts},
+    {{"MSG_RTN_GET_DDID"}, ID_RTN_GET_DDID, hasGFX11Insts},
+    {{"MSG_RTN_GET_TMA"}, ID_RTN_GET_TMA, hasGFX11Insts},
+    {{"MSG_RTN_GET_REALTIME"}, ID_RTN_GET_REALTIME, hasGFX11Insts},
+    {{"MSG_RTN_SAVE_WAVE"}, ID_RTN_SAVE_WAVE, hasGFX11Insts},
+    {{"MSG_RTN_GET_TBA"}, ID_RTN_GET_TBA, hasGFX11Insts},
 };
 // NOLINTEND
 
 const int MSG_SIZE = static_cast<int>(
     sizeof(Msg) / sizeof(CustomOperand<const MCSubtargetInfo &>));
 
 // These two must be in sync with llvm::AMDGPU::SendMsg::Op enum members, see SIDefines.h.
 const char *const OpSysSymbolic[OP_SYS_LAST_] = {

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h

 bool isGFX8Plus(const MCSubtargetInfo &STI);
 bool isGFX9Plus(const MCSubtargetInfo &STI);
 bool isGFX10(const MCSubtargetInfo &STI);
 bool isGFX10Plus(const MCSubtargetInfo &STI);
 bool isNotGFX10Plus(const MCSubtargetInfo &STI);
 bool isGFX10Before1030(const MCSubtargetInfo &STI);
 bool isGFX11(const MCSubtargetInfo &STI);
 bool isGFX11Plus(const MCSubtargetInfo &STI);
+bool hasGFX11Insts(const MCSubtargetInfo &STI);
 bool isNotGFX11Plus(const MCSubtargetInfo &STI);
 bool isGCN3Encoding(const MCSubtargetInfo &STI);
 bool isGFX10_AEncoding(const MCSubtargetInfo &STI);
 bool isGFX10_BEncoding(const MCSubtargetInfo &STI);
 bool hasGFX10_3Insts(const MCSubtargetInfo &STI);
 bool isGFX90A(const MCSubtargetInfo &STI);
 bool isGFX940(const MCSubtargetInfo &STI);
 bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI);

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

 
 //===----------------------------------------------------------------------===//
 // SendMsg
 //===----------------------------------------------------------------------===//
 
 namespace SendMsg {
 
 static uint64_t getMsgIdMask(const MCSubtargetInfo &STI) {
-  return isGFX11Plus(STI) ? ID_MASK_GFX11Plus_ : ID_MASK_PreGFX11_;
+  return hasGFX11Insts(STI) ? ID_MASK_GFX11Plus_ : ID_MASK_PreGFX11_;
 }
 
 int64_t getMsgId(const StringRef Name, const MCSubtargetInfo &STI) {
   int Idx = getOprIdx<const MCSubtargetInfo &>(Name, Msg, MSG_SIZE, STI);
   return (Idx < 0) ? Idx : Msg[Idx].Encoding;
 }
 
 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI) {
bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
                   bool Strict) {
   assert(isValidMsgId(MsgId, STI));
 
   if (!Strict)
     return 0 <= OpId && isUInt<OP_WIDTH_>(OpId);
 
   if (MsgId == ID_SYSMSG)
     return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_;
-  if (!isGFX11Plus(STI)) {
+  if (!hasGFX11Insts(STI)) {
     switch (MsgId) {
     case ID_GS_PreGFX11:
       return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP;
     case ID_GS_DONE_PreGFX11:
       return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_;
     }
   }
   return OpId == OP_NONE_;
 }
 
 StringRef getMsgOpName(int64_t MsgId, int64_t OpId,
                        const MCSubtargetInfo &STI) {
   assert(msgRequiresOp(MsgId, STI));
   return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId];
 }
 
 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,
                       const MCSubtargetInfo &STI, bool Strict) {
   assert(isValidMsgOp(MsgId, OpId, STI, Strict));
 
   if (!Strict)
     return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId);
 
-  if (!isGFX11Plus(STI)) {
+  if (!hasGFX11Insts(STI)) {
     switch (MsgId) {
     case ID_GS_PreGFX11:
       return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_;
     case ID_GS_DONE_PreGFX11:
       return (OpId == OP_GS_NOP) ?
           (StreamId == STREAM_ID_NONE_) :
           (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_);
     }
   }
   return StreamId == STREAM_ID_NONE_;
 }
 
 bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI) {
   return MsgId == ID_SYSMSG ||
-      (!isGFX11Plus(STI) &&
+         (!hasGFX11Insts(STI) &&
           (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11));
 }
 
 bool msgSupportsStream(int64_t MsgId, int64_t OpId,
                        const MCSubtargetInfo &STI) {
-  return !isGFX11Plus(STI) &&
+  return !hasGFX11Insts(STI) &&
          (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11) &&
          OpId != OP_GS_NOP;
 }
 
 void decodeMsg(unsigned Val, uint16_t &MsgId, uint16_t &OpId,
                uint16_t &StreamId, const MCSubtargetInfo &STI) {
   MsgId = Val & getMsgIdMask(STI);
-  if (isGFX11Plus(STI)) {
+  if (hasGFX11Insts(STI)) {
     OpId = 0;
     StreamId = 0;
   } else {
     OpId = (Val & OP_MASK_) >> OP_SHIFT_;
     StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_;
   }
 }
 

 bool isGFX11(const MCSubtargetInfo &STI) {
   return STI.getFeatureBits()[AMDGPU::FeatureGFX11];
 }
 
 bool isGFX11Plus(const MCSubtargetInfo &STI) {
   return isGFX11(STI);
 }
 
+bool hasGFX11Insts(const MCSubtargetInfo &STI) {
+  return STI.getFeatureBits()[AMDGPU::FeatureGFX11Insts];
+}
+
 bool isNotGFX11Plus(const MCSubtargetInfo &STI) {
   return !isGFX11Plus(STI);
 }
 
 bool isNotGFX10Plus(const MCSubtargetInfo &STI) {
   return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI);
 }
 

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sendmsg.rtn.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel=0 -march=amdgcn -mcpu=fiji -mattr="+gfx11-insts" -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX8,GFX8-SDAG %s
 ; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11,GFX11-SDAG %s
+; RUN: llc -global-isel=1 -march=amdgcn -mcpu=fiji -mattr="+gfx11-insts" -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX8,GFX8-GISEL %s
 ; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11,GFX11-GISEL %s
 
+; Note: GFX8 w/ +gfx11-insts is also tested as we want to support selecting those
+;   intrinsics as long as the attribute is present. GPU generation should
+;   not matter.
+
 define amdgpu_kernel void @test_get_doorbell(i32 addrspace(1)* %out) {
+; GFX8-LABEL: test_get_doorbell:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_sendmsg_rtn_b32 s2, sendmsg(MSG_RTN_GET_DOORBELL)
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_store_b32 v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
 ; GFX11-SDAG-LABEL: test_get_doorbell:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
 ; GFX11-SDAG-NEXT:    s_sendmsg_rtn_b32 s2, sendmsg(MSG_RTN_GET_DOORBELL)
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
 ; GFX11-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)

 ; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %ret = call i32 @llvm.amdgcn.s.sendmsg.rtn.i32(i32 128)
   store i32 %ret, i32 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @test_get_ddid(i32 addrspace(1)* %out) {
+; GFX8-LABEL: test_get_ddid:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_sendmsg_rtn_b32 s2, sendmsg(MSG_RTN_GET_DDID)
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_store_b32 v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
 ; GFX11-SDAG-LABEL: test_get_ddid:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
 ; GFX11-SDAG-NEXT:    s_sendmsg_rtn_b32 s2, sendmsg(MSG_RTN_GET_DDID)
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
 ; GFX11-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)

 ; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %ret = call i32 @llvm.amdgcn.s.sendmsg.rtn.i32(i32 129)
   store i32 %ret, i32 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @test_get_tma(i64 addrspace(1)* %out) {
+; GFX8-SDAG-LABEL: test_get_tma:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX8-SDAG-NEXT:    s_sendmsg_rtn_b64 s[2:3], sendmsg(MSG_RTN_GET_TMA)
+; GFX8-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-SDAG-NEXT:    flat_store_b64 v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    s_endpgm
+;
 ; GFX11-LABEL: test_get_tma:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
 ; GFX11-NEXT:    s_sendmsg_rtn_b64 s[2:3], sendmsg(MSG_RTN_GET_TMA)
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX8-GISEL-LABEL: test_get_tma:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX8-GISEL-NEXT:    s_sendmsg_rtn_b64 s[2:3], sendmsg(MSG_RTN_GET_TMA)
+; GFX8-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-GISEL-NEXT:    flat_store_b64 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    s_endpgm
   %ret = call i64 @llvm.amdgcn.s.sendmsg.rtn.i64(i32 130)
   store i64 %ret, i64 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @test_get_realtime(i64 addrspace(1)* %out) {
+; GFX8-SDAG-LABEL: test_get_realtime:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX8-SDAG-NEXT:    s_sendmsg_rtn_b64 s[2:3], sendmsg(MSG_RTN_GET_REALTIME)
+; GFX8-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-SDAG-NEXT:    flat_store_b64 v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    s_endpgm
+;
 ; GFX11-LABEL: test_get_realtime:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
 ; GFX11-NEXT:    s_sendmsg_rtn_b64 s[2:3], sendmsg(MSG_RTN_GET_REALTIME)
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX8-GISEL-LABEL: test_get_realtime:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX8-GISEL-NEXT:    s_sendmsg_rtn_b64 s[2:3], sendmsg(MSG_RTN_GET_REALTIME)
+; GFX8-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-GISEL-NEXT:    flat_store_b64 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    s_endpgm
   %ret = call i64 @llvm.amdgcn.s.sendmsg.rtn.i64(i32 131)
   store i64 %ret, i64 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @test_savewave(i32 addrspace(1)* %out) {
+; GFX8-LABEL: test_savewave:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_sendmsg_rtn_b32 s2, sendmsg(MSG_RTN_SAVE_WAVE)
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_store_b32 v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
 ; GFX11-SDAG-LABEL: test_savewave:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
 ; GFX11-SDAG-NEXT:    s_sendmsg_rtn_b32 s2, sendmsg(MSG_RTN_SAVE_WAVE)
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
 ; GFX11-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)

 ; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %ret = call i32 @llvm.amdgcn.s.sendmsg.rtn.i32(i32 132)
   store i32 %ret, i32 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @test_get_tba(i64 addrspace(1)* %out) {
+; GFX8-SDAG-LABEL: test_get_tba:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX8-SDAG-NEXT:    s_sendmsg_rtn_b64 s[2:3], sendmsg(MSG_RTN_GET_TBA)
+; GFX8-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-SDAG-NEXT:    flat_store_b64 v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    s_endpgm
+;
 ; GFX11-LABEL: test_get_tba:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
 ; GFX11-NEXT:    s_sendmsg_rtn_b64 s[2:3], sendmsg(MSG_RTN_GET_TBA)
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX8-GISEL-LABEL: test_get_tba:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX8-GISEL-NEXT:    s_sendmsg_rtn_b64 s[2:3], sendmsg(MSG_RTN_GET_TBA)
+; GFX8-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-GISEL-NEXT:    flat_store_b64 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    s_endpgm
   %ret = call i64 @llvm.amdgcn.s.sendmsg.rtn.i64(i32 133)
   store i64 %ret, i64 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @test_get_0_i32(i32 addrspace(1)* %out) {
+; GFX8-LABEL: test_get_0_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_sendmsg_rtn_b32 s2, sendmsg(0, 0, 0)
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_store_b32 v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
 ; GFX11-SDAG-LABEL: test_get_0_i32:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
 ; GFX11-SDAG-NEXT:    s_sendmsg_rtn_b32 s2, sendmsg(0, 0, 0)
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
 ; GFX11-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)

 ; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %ret = call i32 @llvm.amdgcn.s.sendmsg.rtn.i32(i32 0)
   store i32 %ret, i32 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @test_get_99999_i64(i64 addrspace(1)* %out) {
+; GFX8-SDAG-LABEL: test_get_99999_i64:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX8-SDAG-NEXT:    s_sendmsg_rtn_b64 s[2:3], 99999
+; GFX8-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-SDAG-NEXT:    flat_store_b64 v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    s_endpgm
+;
 ; GFX11-LABEL: test_get_99999_i64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
 ; GFX11-NEXT:    s_sendmsg_rtn_b64 s[2:3], 99999
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX8-GISEL-LABEL: test_get_99999_i64:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX8-GISEL-NEXT:    s_sendmsg_rtn_b64 s[2:3], 99999
+; GFX8-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-GISEL-NEXT:    flat_store_b64 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    s_endpgm
   %ret = call i64 @llvm.amdgcn.s.sendmsg.rtn.i64(i32 99999)
   store i64 %ret, i64 addrspace(1)* %out
   ret void
 }
 
 declare i32 @llvm.amdgcn.s.sendmsg.rtn.i32(i32)
 declare i64 @llvm.amdgcn.s.sendmsg.rtn.i64(i32)