GlobalISel: moreElementsVector for G_LOAD

[arsenm]

Summary

AMDGPU change and test is a placeholder until a future patch with
complete handling.

[arsenm]

Also store

[arsenm]

Fix store

[arsenm]

Add a comment on G_LOAD extload behavior

[dsanders]

The bit that's here LGTM. Is there another patch that implements moreElementsVectorSrc/moreElementsVectorDst?

include/llvm/Target/GenericOpcodes.td

728–732

Thanks for adding this.

lib/CodeGen/GlobalISel/LegalizerHelper.cpp

2706–2707

Shouldn't we defend against this for the G_STORE case too?

[arsenm]

In D64181#1584131, @dsanders wrote:

The bit that's here LGTM. Is there another patch that implements moreElementsVectorSrc/moreElementsVectorDst?

Those have been around for a while, since around r354345

lib/CodeGen/GlobalISel/LegalizerHelper.cpp

2706–2707

Probably, although I'm not sure it makes sense to ever legalize the pointer operand

[arsenm]

Check type idx for store

[arsenm]

ping

[dsanders]

LGTM

[arsenm]

r367503

include/llvm/Target/GenericOpcodes.td

def G_INTRINSIC_ROUND : GenericInstruction {
   let InOperandList = (ins type0:$src1);
   let hasSideEffects = 0;
 }
 
 //------------------------------------------------------------------------------
 // Memory ops
 //------------------------------------------------------------------------------
 
-// Generic load. Expects a MachineMemOperand in addition to explicit operands.
+// Generic load. Expects a MachineMemOperand in addition to explicit
+// operands. If the result size is larger than the memory size, the
+// high bits are undefined. If the result is a vector type and larger
+// than the memory size, the high elements are undefined (i.e. this is
+// not a per-element, vector anyextload)
 def G_LOAD : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
   let InOperandList = (ins ptype1:$addr);
   let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 // Generic sign-extended load. Expects a MachineMemOperand in addition to explicit operands.

lib/CodeGen/GlobalISel/LegalizerHelper.cpp

 }
 
 LegalizerHelper::LegalizeResult
 LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
                                     LLT MoreTy) {
   MIRBuilder.setInstr(MI);
   unsigned Opc = MI.getOpcode();
   switch (Opc) {
-  case TargetOpcode::G_IMPLICIT_DEF: {
+  case TargetOpcode::G_IMPLICIT_DEF:
+  case TargetOpcode::G_LOAD: {
+    if (TypeIdx != 0)
+      return UnableToLegalize;
     Observer.changingInstr(MI);
     moreElementsVectorDst(MI, MoreTy, 0);
     Observer.changedInstr(MI);
     return Legalized;
   }
+  case TargetOpcode::G_STORE:
+    if (TypeIdx != 0)
+      return UnableToLegalize;
+    Observer.changingInstr(MI);
+    moreElementsVectorSrc(MI, MoreTy, 0);
+    Observer.changedInstr(MI);
+    return Legalized;
   case TargetOpcode::G_AND:
   case TargetOpcode::G_OR:
   case TargetOpcode::G_XOR:
   case TargetOpcode::G_SMIN:
   case TargetOpcode::G_SMAX:
   case TargetOpcode::G_UMIN:
   case TargetOpcode::G_UMAX: {
     Observer.changingInstr(MI);

lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

getActionDefinitionsBuilder({G_LOAD, G_STORE})
     .narrowScalarIf([](const LegalityQuery &Query) {
         unsigned Size = Query.Types[0].getSizeInBits();
         unsigned MemSize = Query.MMODescrs[0].SizeInBits;
         return (Size > 32 && MemSize < Size);
       },
       [](const LegalityQuery &Query) {
         return std::make_pair(0, LLT::scalar(32));
       })
+    .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
     .fewerElementsIf([=](const LegalityQuery &Query) {
         unsigned MemSize = Query.MMODescrs[0].SizeInBits;
         return (MemSize == 96) &&
                Query.Types[0].isVector() &&
                !ST.hasDwordx3LoadStores();
       },
       [=](const LegalityQuery &Query) {
         return std::make_pair(0, V2S32);

test/CodeGen/AMDGPU/GlobalISel/legalize-load.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI %s
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -global-isel-abort=0 %s -o - | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer -global-isel-abort=0  %s -o - | FileCheck -check-prefix=VI %s
 
 ---
 name: test_load_global_i32
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; SI-LABEL: name: test_load_global_i32
bb.0:
     ; VI: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p1) :: (load 12, align 16, addrspace 1)
     ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](<3 x s32>)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<3 x s32>) = G_LOAD %0 :: (load 12, addrspace 1, align 16)
     $vgpr0_vgpr1_vgpr2 = COPY %1
 ...
 
 ---
+
 name: test_load_constant_v8i32
 body: |
   bb.0:
     liveins: $sgpr0_sgpr1
 
     ; CHECK-LABEL: name: test_load_global_v8i32
     ; CHECK: [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
     ; CHECK: [[LOAD:%[0-9]+]]:_(<8 x s32>) = G_LOAD [[COPY]](p4) :: (load 32, addrspace 4)
bb.0:
     ; CHECK-LABEL: name: test_load_global_v16i32
     ; CHECK: [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
     ; CHECK: [[LOAD:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[COPY]](p4) :: (load 64, addrspace 4)
     ; CHECK: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 = COPY [[LOAD]](<16 x s32>)
     %0:_(p4) = COPY $sgpr0_sgpr1
     %1:_(<16 x s32>) = G_LOAD %0 :: (load 64, addrspace 4)
     $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 = COPY %1
 ...
+
+name: test_load_global_v3s8_align4
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_load_global_v3s8_align4
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 3, align 4, addrspace 1)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s8>)
+    ; VI-LABEL: name: test_load_global_v3s8_align4
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[COPY]](p1) :: (load 3, align 4, addrspace 1)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[LOAD]](<4 x s8>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s8>)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<3 x s8>) = G_LOAD %0 :: (load 3, align 4, addrspace 1)
+    S_NOP 0, implicit %1
+...

test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI %s
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -global-isel-abort=0 %s -o - | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer -global-isel-abort=0 %s -o - | FileCheck -check-prefix=VI %s
 ---
 name: test_store_global_i32
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2
 
     ; SI-LABEL: name: test_store_global_i32
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1

 ...
 
 ---
 name: test_truncstore_global_s128_to_s16
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
 
+    ; SI-LABEL: name: test_truncstore_global_s128_to_s16
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s128)
+    ; SI: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store 2, addrspace 1)
+    ; VI-LABEL: name: test_truncstore_global_s128_to_s16
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s128)
+    ; VI: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store 2, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     G_STORE %1, %0 :: (store 2, addrspace 1)
 ...
 
 ---
 name: test_truncstore_global_s128_to_s8
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
 
-    ; SI-LABEL: name: test_truncstore_global_s128_to_s16
+    ; SI-LABEL: name: test_truncstore_global_s128_to_s8
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI: [[COPY1:%[0-9]+]]:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s128)
     ; SI: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
-    ; VI-LABEL: name: test_truncstore_global_s128_to_s16
+    ; VI-LABEL: name: test_truncstore_global_s128_to_s8
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI: [[COPY1:%[0-9]+]]:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s128)
     ; VI: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     G_STORE %1, %0 :: (store 1, addrspace 1)
 ...
bb.0:
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     ; VI: G_STORE [[COPY1]](<2 x s64>), [[COPY]](p1) :: (store 16, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x s64>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     G_STORE %1, %0 :: (store 16, addrspace 1)
 
 ...
+
+---
+name: test_store_global_v3s8_align4
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_store_global_v3s8_align4
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[DEF:%[0-9]+]]:_(<3 x s8>) = G_IMPLICIT_DEF
+    ; SI: [[DEF1:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF1]], [[DEF]](<3 x s8>), 0
+    ; SI: G_STORE [[INSERT]](<4 x s8>), [[COPY]](p1) :: (store 3, align 4, addrspace 1)
+    ; VI-LABEL: name: test_store_global_v3s8_align4
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[DEF:%[0-9]+]]:_(<3 x s8>) = G_IMPLICIT_DEF
+    ; VI: [[DEF1:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF1]], [[DEF]](<3 x s8>), 0
+    ; VI: G_STORE [[INSERT]](<4 x s8>), [[COPY]](p1) :: (store 3, align 4, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<3 x s8>) = G_IMPLICIT_DEF
+    G_STORE %1, %0 :: (store 3, addrspace 1, align 4)
+
+...