[AMDGPU] Add new GFX11 intrinsic llvm.amdgcn.exp.row

[arsenm]

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.row.ll

3

Missing global-isel run line

[foad]

Add globalisel implementation and testing.

[foad]

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.row.ll

3

Good point. The globalisel implementation was not finished.

[foad]

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.row.ll

3

Fixed now. Please take another look.

llvm/include/llvm/IR/IntrinsicsAMDGPU.td

def int_amdgcn_exp : Intrinsic <[], [
   llvm_i1_ty,        // done
   llvm_i1_ty         // vm
   ],
   [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<6>>,
    ImmArg<ArgIndex<7>>, IntrWriteMem, IntrInaccessibleMemOnly,
    IntrWillReturn]
 >;
 
-// exp with compr bit set.
+// exp with row_en bit set. Only supported on GFX11+.
+def int_amdgcn_exp_row : Intrinsic <[], [
+  llvm_i32_ty,       // tgt,
+  llvm_i32_ty,       // en
+  llvm_any_ty,       // src0 (f32 or i32)
+  LLVMMatchType<0>,  // src1
+  LLVMMatchType<0>,  // src2
+  LLVMMatchType<0>,  // src3
+  llvm_i1_ty,        // done
+  llvm_i32_ty],      // row number
+  [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<6>>,
+   IntrWriteMem, IntrInaccessibleMemOnly, IntrWillReturn]
+>;
+
+// exp with compr bit set. Not supported on GFX11+.
 def int_amdgcn_exp_compr : Intrinsic <[], [
   llvm_i32_ty,       // tgt,
   llvm_i32_ty,       // en
   llvm_anyvector_ty, // src0 (v2f16 or v2i16)
   LLVMMatchType<0>,  // src1
   llvm_i1_ty,        // done
   llvm_i1_ty],       // vm
   [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<4>>,

llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp

case Intrinsic::amdgcn_sbfe: {
 
     Value *RightShift = Signed ? IC.Builder.CreateAShr(Src, Offset)
                                : IC.Builder.CreateLShr(Src, Offset);
 
     RightShift->takeName(&II);
     return IC.replaceInstUsesWith(II, RightShift);
   }
   case Intrinsic::amdgcn_exp:
+  case Intrinsic::amdgcn_exp_row:
   case Intrinsic::amdgcn_exp_compr: {
     ConstantInt *En = cast<ConstantInt>(II.getArgOperand(1));
     unsigned EnBits = En->getZExtValue();
     if (EnBits == 0xf)
       break; // All inputs enabled.
 
     bool IsCompr = IID == Intrinsic::amdgcn_exp_compr;
     bool Changed = false;

llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp

case Intrinsic::amdgcn_struct_buffer_load_lds: {
       constrainOpWithReadfirstlane(MI, MRI, 6); // soffset
       return;
     }
     case Intrinsic::amdgcn_global_load_lds: {
       applyDefaultMapping(OpdMapper);
       constrainOpWithReadfirstlane(MI, MRI, 2);
       return;
     }
+    case Intrinsic::amdgcn_exp_row:
+      applyDefaultMapping(OpdMapper);
+      constrainOpWithReadfirstlane(MI, MRI, 8); // M0
+      return;
     default: {
       if (const AMDGPU::RsrcIntrinsic *RSrcIntrin =
               AMDGPU::lookupRsrcIntrinsic(IntrID)) {
         // Non-images can have complications from operands that allow both SGPR
         // and VGPR. For now it's too complicated to figure out the final opcode
         // to derive the register bank from the MCInstrDesc.
         if (RSrcIntrin->IsImage) {
           applyMappingImage(MI, OpdMapper, MRI, RSrcIntrin->RsrcArg);
case Intrinsic::amdgcn_exp_compr:
       break;
     case Intrinsic::amdgcn_exp:
       // FIXME: Could we support packed types here?
       OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
       OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
       OpdsMapping[5] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
       OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
       break;
+    case Intrinsic::amdgcn_exp_row:
+      OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
+      OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
+      OpdsMapping[5] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
+      OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
+      OpdsMapping[8] = getSGPROpMapping(MI.getOperand(8).getReg(), MRI, *TRI);
+      break;
     case Intrinsic::amdgcn_s_sendmsg:
     case Intrinsic::amdgcn_s_sendmsghalt: {
       // This must be an SGPR, but accept a VGPR.
       unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
                                    AMDGPU::SGPRRegBankID);
       OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
       break;
     }

llvm/lib/Target/AMDGPU/EXPInstructions.td

class ExpPattern<ValueType vt, Instruction Inst, int done_val> : GCNPat<
   (int_amdgcn_exp timm:$tgt, timm:$en,
                   (vt ExpSrc0:$src0), (vt ExpSrc1:$src1),
                   (vt ExpSrc2:$src2), (vt ExpSrc3:$src3),
                   done_val, timm:$vm),
   (Inst timm:$tgt, ExpSrc0:$src0, ExpSrc1:$src1,
         ExpSrc2:$src2, ExpSrc3:$src3, timm:$vm, 0, timm:$en)
 >;
 
+class ExpRowPattern<ValueType vt, Instruction Inst, int done_val> : GCNPat<
+  (int_amdgcn_exp_row timm:$tgt, timm:$en,
+                      (vt ExpSrc0:$src0), (vt ExpSrc1:$src1),
+                      (vt ExpSrc2:$src2), (vt ExpSrc3:$src3),
+                      done_val, M0),
+  (Inst timm:$tgt, ExpSrc0:$src0, ExpSrc1:$src1,
+        ExpSrc2:$src2, ExpSrc3:$src3, 0, 0, timm:$en)
+>;
+
 class ExpComprPattern<ValueType vt, Instruction Inst, int done_val> : GCNPat<
   (int_amdgcn_exp_compr timm:$tgt, timm:$en,
                         (vt ExpSrc0:$src0), (vt ExpSrc1:$src1),
                         done_val, timm:$vm),
   (Inst timm:$tgt, ExpSrc0:$src0, ExpSrc1:$src1,
         (IMPLICIT_DEF), (IMPLICIT_DEF), timm:$vm, 1, timm:$en)
 >;
 
 // FIXME: The generated DAG matcher seems to have strange behavior
 // with a 1-bit literal to match, so use a -1 for checking a true
 // 1-bit value.
 def : ExpPattern<i32, EXP, 0>;
 def : ExpPattern<i32, EXP_DONE, -1>;
 def : ExpPattern<f32, EXP, 0>;
 def : ExpPattern<f32, EXP_DONE, -1>;
 
+def : ExpRowPattern<i32, EXP_ROW, 0>;
+def : ExpRowPattern<i32, EXP_ROW_DONE, -1>;
+def : ExpRowPattern<f32, EXP_ROW, 0>;
+def : ExpRowPattern<f32, EXP_ROW_DONE, -1>;
+
 def : ExpComprPattern<v2i16, EXP, 0>;
 def : ExpComprPattern<v2i16, EXP_DONE, -1>;
 def : ExpComprPattern<v2f16, EXP, 0>;
 def : ExpComprPattern<v2f16, EXP_DONE, -1>;

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.row.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX11,GFX11-SDAG
+; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL
+
+declare void @llvm.amdgcn.exp.row.i32(i32, i32, i32, i32, i32, i32, i1, i32)
+declare void @llvm.amdgcn.exp.row.f32(i32, i32, float, float, float, float, i1, i32)
+declare i32 @llvm.amdgcn.workitem.id.x()
+
+define amdgpu_kernel void @undef_i32() #0 {
+; GFX11-LABEL: undef_i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    exp pos0 off, off, off, off row_en
+; GFX11-NEXT:    exp pos1 off, off, off, off done row_en
+; GFX11-NEXT:    s_endpgm
+  call void @llvm.amdgcn.exp.row.i32(i32 12, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i1 false, i32 0)
+  call void @llvm.amdgcn.exp.row.i32(i32 13, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i1 true, i32 0)
+  ret void
+}
+
+define amdgpu_kernel void @undef_f32() #0 {
+; GFX11-LABEL: undef_f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    exp pos0 off, off, off, off row_en
+; GFX11-NEXT:    exp pos1 off, off, off, off done row_en
+; GFX11-NEXT:    s_endpgm
+  call void @llvm.amdgcn.exp.row.f32(i32 12, i32 0, float undef, float undef, float undef, float undef, i1 false, i32 0)
+  call void @llvm.amdgcn.exp.row.f32(i32 13, i32 0, float undef, float undef, float undef, float undef, i1 true, i32 0)
+  ret void
+}
+
+define amdgpu_kernel void @zero_i32() #0 {
+; GFX11-LABEL: zero_i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    exp pos0 v0, v0, v0, off row_en
+; GFX11-NEXT:    exp pos1 v0, v0, v0, off done row_en
+; GFX11-NEXT:    s_endpgm
+  call void @llvm.amdgcn.exp.row.i32(i32 12, i32 7, i32 0, i32 0, i32 0, i32 undef, i1 false, i32 0)
+  call void @llvm.amdgcn.exp.row.i32(i32 13, i32 7, i32 0, i32 0, i32 0, i32 undef, i1 true, i32 0)
+  ret void
+}
+
+define amdgpu_kernel void @one_f32() #0 {
+; GFX11-LABEL: one_f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_mov_b32_e32 v0, 1.0
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    exp pos0 v0, v0, v0, off row_en
+; GFX11-NEXT:    exp pos1 v0, v0, v0, off done row_en
+; GFX11-NEXT:    s_endpgm
+  call void @llvm.amdgcn.exp.row.f32(i32 12, i32 7, float 1.0, float 1.0, float 1.0, float undef, i1 false, i32 0)
+  call void @llvm.amdgcn.exp.row.f32(i32 13, i32 7, float 1.0, float 1.0, float 1.0, float undef, i1 true, i32 0)
+  ret void
+}
+
+define amdgpu_kernel void @id_i32() #0 {
+; GFX11-LABEL: id_i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    exp pos0 v0, off, off, off done row_en
+; GFX11-NEXT:    s_endpgm
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  call void @llvm.amdgcn.exp.row.i32(i32 12, i32 1, i32 %id, i32 undef, i32 undef, i32 undef, i1 true, i32 0)
+  ret void
+}
+
+define amdgpu_kernel void @id_arg_i32(i32 %row) #0 {
+; GFX11-LABEL: id_arg_i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b32 s0, s[0:1], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 m0, s0
+; GFX11-NEXT:    exp pos0 v0, off, off, off done row_en
+; GFX11-NEXT:    s_endpgm
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  call void @llvm.amdgcn.exp.row.i32(i32 12, i32 1, i32 %id, i32 undef, i32 undef, i32 undef, i1 true, i32 %row)
+  ret void
+}
+
+; Divergent row number just causes a readfirstlane for now.
+define amdgpu_kernel void @id_row_i32() #0 {
+; GFX11-SDAG-LABEL: id_row_i32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v0, 0x63
+; GFX11-SDAG-NEXT:    s_mov_b32 m0, s0
+; GFX11-SDAG-NEXT:    exp pos0 v0, off, off, off done row_en
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: id_row_i32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v1, 0x63
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 m0, v0
+; GFX11-GISEL-NEXT:    exp pos0 v1, off, off, off done row_en
+; GFX11-GISEL-NEXT:    s_endpgm
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  call void @llvm.amdgcn.exp.row.i32(i32 12, i32 1, i32 99, i32 undef, i32 undef, i32 undef, i1 true, i32 %id)
+  ret void
+}