This is an archive of the discontinued LLVM Phabricator instance.

Differential D72796

AMDGPU/GlobalISel: Legalize unpacked d16 image operations
ClosedPublic

Authored by arsenm on Jan 15 2020, 11:50 AM.

Details

Reviewers
nhaehnle
kerbowa
Summary

On targets that don't have the normal packed f16 layout, handle these
during legalization. Directly modify the register types. We can infer
this was a d16 load based on the mem operand size during selection.

A16 operands should possibly be handled here as well, but don't worry
about that yet.

Diff Detail

Event Timeline

arsenm created this revision.Jan 15 2020, 11:50 AM
Herald added a project: Restricted Project. · View Herald TranscriptJan 15 2020, 11:50 AM
Herald added subscribers: Petar.Avramovic, jfb, hiraditya and 8 others. · View Herald Transcript
arsenm added a child revision: D73444: AMDGPU/GlobalISel: Legalize TFE image result loads.Jan 26 2020, 7:51 PM
nhaehnle added a comment.Jan 27 2020, 8:56 AM

This seems largely reasonable to me, but I do think the observer notification needs to happen.

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
2731–2734

It might help the readability of legalizeIntrinsic to move this test there and pass the ImageDimIntr in?

2754

I think so?

arsenm marked an inline comment as done.Jan 27 2020, 9:07 AM
arsenm added inline comments.
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
2754

legalizeIntrinsic doesn't have access to the observer, so I'll have to add that and audit the rest of these lowering at some point

arsenm updated this revision to Diff 240614.Jan 27 2020, 9:24 AM

Move ImageDimIntrinsicInfo check

arsenm added a child revision: D73666: AMDGPU/GlobalISel: Adjust image load register type based on dmask.Jan 29 2020, 2:31 PM
nhaehnle accepted this revision.Jan 30 2020, 1:49 AM

Thanks, LGTM modulo the observer question -- I tend to think it's necessary, but I'll defer to your judgment.

This revision is now accepted and ready to land.Jan 30 2020, 1:49 AM
arsenm added a comment.Jan 30 2020, 5:38 AM
In D72796#1848904, @nhaehnle wrote:

Thanks, LGTM modulo the observer question -- I tend to think it's necessary, but I'll defer to your judgment.

In D72796#1848904, @nhaehnle wrote:

Thanks, LGTM modulo the observer question -- I tend to think it's necessary, but I'll defer to your judgment.

I added the needed observer argument in c5fffa4da35f0fcc89b5ea88cc1bc60bc475a18e, so I've fixed this.

arsenm closed this revision.Jan 30 2020, 5:38 AM

872e899b7563a033228be935a04bfb853bd73acd

Revision Contents

PathSize
llvm/
lib/
Target/
AMDGPU/
3 lines
65 lines
test/
CodeGen/
AMDGPU/
GlobalISel/
201 lines
205 lines

Diff 238326

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h

Show First 20 Lines • Show All 113 Lines • ▼ Show 20 Lines bool legalizeBufferStore(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B, bool IsTyped, MachineIRBuilder &B, bool IsTyped,
bool IsFormat) const; bool IsFormat) const;
bool legalizeBufferLoad(MachineInstr &MI, MachineRegisterInfo &MRI, bool legalizeBufferLoad(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B, bool IsTyped, MachineIRBuilder &B, bool IsTyped,
bool IsFormat) const; bool IsFormat) const;
bool legalizeBufferAtomic(MachineInstr &MI, MachineIRBuilder &B, bool legalizeBufferAtomic(MachineInstr &MI, MachineIRBuilder &B,
Intrinsic::ID IID) const; Intrinsic::ID IID) const;
bool legalizeImageIntrinsic(MachineInstr &MI, MachineIRBuilder &B,
Intrinsic::ID IID) const;
bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const override; MachineIRBuilder &B) const override;
}; };
} // End llvm namespace. } // End llvm namespace.
#endif #endif

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

Show First 20 Lines • Show All 2,714 Lines • ▼ Show 20 Lines MIB.addUse(RSrc) // rsrc
.addImm(AuxiliaryData) // cachepolicy, swizzled buffer(imm) .addImm(AuxiliaryData) // cachepolicy, swizzled buffer(imm)
.addImm(HasVIndex ? -1 : 0) // idxen(imm) .addImm(HasVIndex ? -1 : 0) // idxen(imm)
.addMemOperand(MMO); .addMemOperand(MMO);
MI.eraseFromParent(); MI.eraseFromParent();
return true; return true;
} }
bool AMDGPULegalizerInfo::legalizeImageIntrinsic(MachineInstr &MI,
MachineIRBuilder &B,
Intrinsic::ID IID) const {
// We are only processing the operands of d16 image operations on subtargets
// that use the unpacked register layout.
if (!ST.hasUnpackedD16VMem())
return true;
const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
AMDGPU::getImageDimIntrinsicInfo(IID);
if (!ImageDimIntr)
return true; // Other intrinsic to ignore
nhaehnleUnsubmitted
Not Done
ReplyInline Actions

It might help the readability of legalizeIntrinsic to move this test there and pass the ImageDimIntr in?

nhaehnle: It might help the readability of legalizeIntrinsic to move this test there and pass the…
const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
AMDGPU::getMIMGBaseOpcodeInfo(ImageDimIntr->BaseOpcode);
if (BaseOpcode->Atomic) // No d16 atomics
return true;
MachineRegisterInfo *MRI = B.getMRI();
const LLT S32 = LLT::scalar(32);
const LLT S16 = LLT::scalar(16);
if (BaseOpcode->Store) {
Register VData = MI.getOperand(1).getReg();
LLT Ty = MRI->getType(VData);
if (!Ty.isVector() || Ty.getElementType() != S16)
return true;
B.setInstr(MI);
// FIXME: Do we need to notify the observer of the instruction change?
nhaehnleUnsubmitted
Not Done
ReplyInline Actions

I think so?

nhaehnle: I think so?
arsenmAuthorUnsubmitted
Done
ReplyInline Actions

legalizeIntrinsic doesn't have access to the observer, so I'll have to add that and audit the rest of these lowering at some point

arsenm: legalizeIntrinsic doesn't have access to the observer, so I'll have to add that and audit the…
MI.getOperand(1).setReg(handleD16VData(B, *MRI, VData));
return true;
}
// Must be an image load.
Register DstReg = MI.getOperand(0).getReg();
LLT Ty = MRI->getType(DstReg);
if (!Ty.isVector() || Ty.getElementType() != S16)
return true;
B.setInsertPt(*MI.getParent(), ++MI.getIterator());
LLT WidenedTy = Ty.changeElementType(S32);
Register WideDstReg = MRI->createGenericVirtualRegister(WidenedTy);
// FIXME: Do we need to notify the observer of the instruction change?
MI.getOperand(0).setReg(WideDstReg);
// FIXME: Just vector trunc should be sufficent, but legalization currently
// broken.
auto Unmerge = B.buildUnmerge(S32, WideDstReg);
int NumOps = Unmerge->getNumOperands() - 1;
SmallVector<Register, 4> RemergeParts(NumOps);
for (int I = 0; I != NumOps; ++I)
RemergeParts[I] = B.buildTrunc(S16, Unmerge.getReg(I)).getReg(0);
B.buildBuildVector(DstReg, RemergeParts);
return true;
}
bool AMDGPULegalizerInfo::legalizeIntrinsic(MachineInstr &MI, bool AMDGPULegalizerInfo::legalizeIntrinsic(MachineInstr &MI,
MachineRegisterInfo &MRI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const { MachineIRBuilder &B) const {
// Replace the use G_BRCOND with the exec manipulate and branch pseudos. // Replace the use G_BRCOND with the exec manipulate and branch pseudos.
auto IntrID = MI.getIntrinsicID(); auto IntrID = MI.getIntrinsicID();
switch (IntrID) { switch (IntrID) {
case Intrinsic::amdgcn_if: case Intrinsic::amdgcn_if:
case Intrinsic::amdgcn_else: { case Intrinsic::amdgcn_else: {
▲ Show 20 Lines • Show All 144 Lines • ▼ Show 20 Linesbool AMDGPULegalizerInfo::legalizeIntrinsic(MachineInstr &MI,
case Intrinsic::amdgcn_raw_buffer_atomic_inc: case Intrinsic::amdgcn_raw_buffer_atomic_inc:
case Intrinsic::amdgcn_struct_buffer_atomic_inc: case Intrinsic::amdgcn_struct_buffer_atomic_inc:
case Intrinsic::amdgcn_raw_buffer_atomic_dec: case Intrinsic::amdgcn_raw_buffer_atomic_dec:
case Intrinsic::amdgcn_struct_buffer_atomic_dec: case Intrinsic::amdgcn_struct_buffer_atomic_dec:
case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap: case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap: case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
return legalizeBufferAtomic(MI, B, IntrID); return legalizeBufferAtomic(MI, B, IntrID);
default: default:
return true; return legalizeImageIntrinsic(MI, B, IntrID);
} }
return true; return true;
} }

llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll

  • This file was added.
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -stop-after=legalizer -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=legalizer -o - %s | FileCheck -check-prefix=PACKED %s
define amdgpu_ps half @image_load_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; UNPACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; UNPACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; UNPACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; UNPACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; UNPACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; UNPACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; UNPACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; UNPACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; UNPACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; UNPACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; UNPACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; UNPACKED: [[INT:%[0-9]+]]:_(s16) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.2d), 1, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
; UNPACKED: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[INT]](s16)
; UNPACKED: $vgpr0 = COPY [[ANYEXT]](s32)
; UNPACKED: SI_RETURN_TO_EPILOG implicit $vgpr0
; PACKED-LABEL: name: image_load_f16
; PACKED: bb.1 (%ir-block.0):
; PACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; PACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; PACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; PACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; PACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; PACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; PACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; PACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; PACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; PACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; PACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; PACKED: [[INT:%[0-9]+]]:_(s16) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.2d), 1, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
; PACKED: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[INT]](s16)
; PACKED: $vgpr0 = COPY [[ANYEXT]](s32)
; PACKED: SI_RETURN_TO_EPILOG implicit $vgpr0
%tex = call half @llvm.amdgcn.image.load.2d.f16.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret half %tex
}
define amdgpu_ps <2 x half> @image_load_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_v2f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; UNPACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; UNPACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; UNPACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; UNPACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; UNPACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; UNPACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; UNPACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; UNPACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; UNPACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; UNPACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; UNPACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; UNPACKED: [[INT:%[0-9]+]]:_(<2 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.2d), 3, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
; UNPACKED: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[INT]](<2 x s32>)
; UNPACKED: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
; UNPACKED: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
; UNPACKED: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; UNPACKED: $vgpr0 = COPY [[BUILD_VECTOR1]](<2 x s16>)
; UNPACKED: SI_RETURN_TO_EPILOG implicit $vgpr0
; PACKED-LABEL: name: image_load_v2f16
; PACKED: bb.1 (%ir-block.0):
; PACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; PACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; PACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; PACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; PACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; PACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; PACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; PACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; PACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; PACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; PACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; PACKED: [[INT:%[0-9]+]]:_(<2 x s16>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.2d), 3, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
; PACKED: $vgpr0 = COPY [[INT]](<2 x s16>)
; PACKED: SI_RETURN_TO_EPILOG implicit $vgpr0
%tex = call <2 x half> @llvm.amdgcn.image.load.2d.v2f16.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret <2 x half> %tex
}
define amdgpu_ps <3 x half> @image_load_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_v3f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; UNPACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; UNPACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; UNPACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; UNPACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; UNPACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; UNPACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; UNPACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; UNPACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; UNPACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; UNPACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; UNPACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; UNPACKED: [[INT:%[0-9]+]]:_(<3 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.2d), 7, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
; UNPACKED: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[INT]](<3 x s32>)
; UNPACKED: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
; UNPACKED: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
; UNPACKED: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV2]](s32)
; UNPACKED: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; UNPACKED: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; UNPACKED: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
; UNPACKED: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
; UNPACKED: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
; UNPACKED: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
; UNPACKED: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
; UNPACKED: [[EXTRACT1:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 0
; UNPACKED: [[EXTRACT2:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 32
; UNPACKED: $vgpr0 = COPY [[EXTRACT1]](<2 x s16>)
; UNPACKED: $vgpr1 = COPY [[EXTRACT2]](<2 x s16>)
; UNPACKED: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
; PACKED-LABEL: name: image_load_v3f16
; PACKED: bb.1 (%ir-block.0):
; PACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; PACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; PACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; PACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; PACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; PACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; PACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; PACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; PACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; PACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; PACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; PACKED: [[INT:%[0-9]+]]:_(<3 x s16>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.2d), 7, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
; PACKED: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
; PACKED: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[INT]](<3 x s16>), 0
; PACKED: [[EXTRACT:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 0
; PACKED: [[EXTRACT1:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 32
; PACKED: $vgpr0 = COPY [[EXTRACT]](<2 x s16>)
; PACKED: $vgpr1 = COPY [[EXTRACT1]](<2 x s16>)
; PACKED: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
%tex = call <3 x half> @llvm.amdgcn.image.load.2d.v3f16.i32(i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret <3 x half> %tex
}
define amdgpu_ps <4 x half> @image_load_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
; UNPACKED-LABEL: name: image_load_v4f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; UNPACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; UNPACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; UNPACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; UNPACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; UNPACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; UNPACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; UNPACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; UNPACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; UNPACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; UNPACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; UNPACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; UNPACKED: [[INT:%[0-9]+]]:_(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.2d), 15, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
; UNPACKED: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[INT]](<4 x s32>)
; UNPACKED: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
; UNPACKED: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
; UNPACKED: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV2]](s32)
; UNPACKED: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[UV3]](s32)
; UNPACKED: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; UNPACKED: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
; UNPACKED: $vgpr0 = COPY [[BUILD_VECTOR1]](<2 x s16>)
; UNPACKED: $vgpr1 = COPY [[BUILD_VECTOR2]](<2 x s16>)
; UNPACKED: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
; PACKED-LABEL: name: image_load_v4f16
; PACKED: bb.1 (%ir-block.0):
; PACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
; PACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; PACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; PACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; PACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; PACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; PACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; PACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; PACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; PACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; PACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; PACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; PACKED: [[INT:%[0-9]+]]:_(<4 x s16>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.2d), 15, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
; PACKED: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INT]](<4 x s16>)
; PACKED: $vgpr0 = COPY [[UV]](<2 x s16>)
; PACKED: $vgpr1 = COPY [[UV1]](<2 x s16>)
; PACKED: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
%tex = call <4 x half> @llvm.amdgcn.image.load.2d.v4f16.i32(i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret <4 x half> %tex
}
declare half @llvm.amdgcn.image.load.2d.f16.i32(i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
declare <2 x half> @llvm.amdgcn.image.load.2d.v2f16.i32(i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
declare <3 x half> @llvm.amdgcn.image.load.2d.v3f16.i32(i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
declare <4 x half> @llvm.amdgcn.image.load.2d.v4f16.i32(i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
attributes #0 = { nounwind readonly }

llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll

  • This file was added.
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -stop-after=legalizer -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=legalizer -o - %s | FileCheck -check-prefix=PACKED %s
define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, half %data) {
; UNPACKED-LABEL: name: image_store_f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
; UNPACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; UNPACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; UNPACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; UNPACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; UNPACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; UNPACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; UNPACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; UNPACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; UNPACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; UNPACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; UNPACKED: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
; UNPACKED: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
; UNPACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; UNPACKED: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](s16), 1, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 2 into custom "TargetCustom8")
; UNPACKED: S_ENDPGM 0
; PACKED-LABEL: name: image_store_f16
; PACKED: bb.1 (%ir-block.0):
; PACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
; PACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; PACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; PACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; PACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; PACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; PACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; PACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; PACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; PACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; PACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; PACKED: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
; PACKED: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
; PACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; PACKED: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](s16), 1, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 2 into custom "TargetCustom8")
; PACKED: S_ENDPGM 0
call void @llvm.amdgcn.image.store.2d.f16.i32(half %data, i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret void
}
define amdgpu_ps void @image_store_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <2 x half> %in) {
; UNPACKED-LABEL: name: image_store_v2f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
; UNPACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; UNPACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; UNPACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; UNPACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; UNPACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; UNPACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; UNPACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; UNPACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; UNPACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; UNPACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; UNPACKED: [[COPY10:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
; UNPACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; UNPACKED: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x s16>)
; UNPACKED: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; UNPACKED: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
; UNPACKED: [[COPY11:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
; UNPACKED: [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
; UNPACKED: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY11]](s32), [[COPY12]](s32)
; UNPACKED: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR1]](<2 x s32>), 3, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 4 into custom "TargetCustom8")
; UNPACKED: S_ENDPGM 0
; PACKED-LABEL: name: image_store_v2f16
; PACKED: bb.1 (%ir-block.0):
; PACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
; PACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; PACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; PACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; PACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; PACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; PACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; PACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; PACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; PACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; PACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; PACKED: [[COPY10:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
; PACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; PACKED: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.store.2d), [[COPY10]](<2 x s16>), 3, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 4 into custom "TargetCustom8")
; PACKED: S_ENDPGM 0
call void @llvm.amdgcn.image.store.2d.v2f16.i32(<2 x half> %in, i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret void
}
define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <3 x half> %in) {
; UNPACKED-LABEL: name: image_store_v3f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; UNPACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; UNPACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; UNPACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; UNPACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; UNPACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; UNPACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; UNPACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; UNPACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; UNPACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; UNPACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; UNPACKED: [[COPY10:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
; UNPACKED: [[COPY11:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
; UNPACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; UNPACKED: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[COPY10]](<2 x s16>), [[COPY11]](<2 x s16>)
; UNPACKED: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
; UNPACKED: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
; UNPACKED: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
; UNPACKED: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
; UNPACKED: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
; UNPACKED: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; UNPACKED: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
; UNPACKED: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
; UNPACKED: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
; UNPACKED: [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
; UNPACKED: [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
; UNPACKED: [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
; UNPACKED: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32)
; UNPACKED: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR1]](<3 x s32>), 7, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 6 into custom "TargetCustom8", align 8)
; UNPACKED: S_ENDPGM 0
; PACKED-LABEL: name: image_store_v3f16
; PACKED: bb.1 (%ir-block.0):
; PACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; PACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; PACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; PACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; PACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; PACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; PACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; PACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; PACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; PACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; PACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; PACKED: [[COPY10:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
; PACKED: [[COPY11:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
; PACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; PACKED: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[COPY10]](<2 x s16>), [[COPY11]](<2 x s16>)
; PACKED: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
; PACKED: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.store.2d), [[EXTRACT]](<3 x s16>), 7, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 6 into custom "TargetCustom8", align 8)
; PACKED: S_ENDPGM 0
call void @llvm.amdgcn.image.store.2d.v3f16.i32(<3 x half> %in, i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret void
}
define amdgpu_ps void @image_store_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <4 x half> %in) {
; UNPACKED-LABEL: name: image_store_v4f16
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; UNPACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; UNPACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; UNPACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; UNPACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; UNPACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; UNPACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; UNPACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; UNPACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; UNPACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; UNPACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; UNPACKED: [[COPY10:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
; UNPACKED: [[COPY11:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
; UNPACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; UNPACKED: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x s16>)
; UNPACKED: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; UNPACKED: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
; UNPACKED: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY11]](<2 x s16>)
; UNPACKED: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
; UNPACKED: [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
; UNPACKED: [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
; UNPACKED: [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
; UNPACKED: [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
; UNPACKED: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
; UNPACKED: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 8 into custom "TargetCustom8")
; UNPACKED: S_ENDPGM 0
; PACKED-LABEL: name: image_store_v4f16
; PACKED: bb.1 (%ir-block.0):
; PACKED: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; PACKED: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
; PACKED: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
; PACKED: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
; PACKED: [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
; PACKED: [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
; PACKED: [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
; PACKED: [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
; PACKED: [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
; PACKED: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
; PACKED: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
; PACKED: [[COPY10:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
; PACKED: [[COPY11:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
; PACKED: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
; PACKED: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[COPY10]](<2 x s16>), [[COPY11]](<2 x s16>)
; PACKED: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.store.2d), [[CONCAT_VECTORS]](<4 x s16>), 15, [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 8 into custom "TargetCustom8")
; PACKED: S_ENDPGM 0
call void @llvm.amdgcn.image.store.2d.v4f16.i32(<4 x half> %in, i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret void
}
declare void @llvm.amdgcn.image.store.2d.f16.i32(half, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
declare void @llvm.amdgcn.image.store.2d.v2f16.i32(<2 x half>, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
declare void @llvm.amdgcn.image.store.2d.v3f16.i32(<3 x half>, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
declare void @llvm.amdgcn.image.store.2d.v4f16.i32(<4 x half>, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
attributes #0 = { nounwind writeonly }