[AMDGPU] Use three- and five-dword result type in image ops

[tpr]

Summary

Some image ops return three or five dwords. Previously, we modeled that
with a 4 or 8 dword register class. The register allocator could
cleverly spot that some subregs were dead and allocate something else
there, but that caused the de-optimization that waitcnt insertion would
think that the result was used immediately.

This commit allows such an image op to have a result with a three or
five dword result, avoiding the above de-optimization.

Change-Id: I3651211bbd7ed22721ee7b9fefd7bcc60a809d8b

[tpr]

Ping.

[arsenm]

LGTM

lib/Target/AMDGPU/MIMGInstructions.td

let BaseOpcode = !cast<MIMGBaseOpcode>(NAME),
     let VDataDwords = 1 in
     defm _V1 : MIMG_NoSampler_Src_Helper <op, asm, VGPR_32, 1>;
     let VDataDwords = 2 in
     defm _V2 : MIMG_NoSampler_Src_Helper <op, asm, VReg_64, 0>;
     let VDataDwords = 3 in
     defm _V3 : MIMG_NoSampler_Src_Helper <op, asm, VReg_96, 0>;
     let VDataDwords = 4 in
     defm _V4 : MIMG_NoSampler_Src_Helper <op, asm, VReg_128, 0>;
-    let VDataDwords = 8 in
-    defm _V8 : MIMG_NoSampler_Src_Helper <op, asm, VReg_256, 0>;
+    let VDataDwords = 5 in
+    defm _V5 : MIMG_NoSampler_Src_Helper <op, asm, VReg_160, 0>;
   }
 }
 
 class MIMG_Store_Helper <bits<7> op, string asm,
                          RegisterClass data_rc,
                          RegisterClass addr_rc,
                          string dns = "">
   : MIMG <(outs), dns>,
let BaseOpcode = !cast<MIMGBaseOpcode>(NAME), WQM = wqm,
     let VDataDwords = 1 in
     defm _V1 : MIMG_Sampler_Src_Helper<op, asm, sample, VGPR_32, 1>;
     let VDataDwords = 2 in
     defm _V2 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_64>;
     let VDataDwords = 3 in
     defm _V3 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_96>;
     let VDataDwords = 4 in
     defm _V4 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_128>;
-    let VDataDwords = 8 in
-    defm _V8 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_256>;
+    let VDataDwords = 5 in
+    defm _V5 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_160>;
   }
 }
 
 multiclass MIMG_Sampler_WQM <bits<7> op, AMDGPUSampleVariant sample>
     : MIMG_Sampler<op, sample, 1>;
 
 multiclass MIMG_Gather <bits<7> op, AMDGPUSampleVariant sample, bit wqm = 0,
                         string asm = "image_gather4"#sample.LowerCaseMod> {
   def "" : MIMG_Sampler_BaseOpcode<sample> {
     let HasD16 = 1;
     let Gather4 = 1;
   }
 
   let BaseOpcode = !cast<MIMGBaseOpcode>(NAME), WQM = wqm,
       Gather4 = 1, hasPostISelHook = 0 in {
     let VDataDwords = 2 in
     defm _V2 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_64>; /* for packed D16 only */
     let VDataDwords = 4 in
     defm _V4 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_128, 1>;
-    let VDataDwords = 8 in
-    defm _V8 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_256>;
+    let VDataDwords = 5 in
+    defm _V5 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_160>;
   }
 }
 
 multiclass MIMG_Gather_WQM <bits<7> op, AMDGPUSampleVariant sample>
     : MIMG_Gather<op, sample, 1>;
 
 //===----------------------------------------------------------------------===//
 // MIMG Instructions

lib/Target/AMDGPU/SIISelLowering.cpp

static SDValue constructRetValue(SelectionDAG &DAG,
   if (IsD16 && !Unpacked)
     NumElts = NumVDataDwords << 1;
   else
     NumElts = NumVDataDwords;
 
   EVT CastVT = NumElts > 1 ? EVT::getVectorVT(Context, AdjEltVT, NumElts)
                            : AdjEltVT;
 
-  // Special case for v8f16. Rather than add support for this, use v4i32 to
+  // Special case for v6f16. Rather than add support for this, use v3i32 to
   // extract the data elements
-  bool V8F16Special = false;
-  if (CastVT == MVT::v8f16) {
-    CastVT = MVT::v4i32;
+  bool V6F16Special = false;
+  if (NumElts == 6) {
+    CastVT = EVT::getVectorVT(Context, MVT::i32, NumElts / 2);
     DMaskPop >>= 1;
     ReqRetNumElts >>= 1;
-    V8F16Special = true;
+    V6F16Special = true;
     AdjVT = MVT::v2i32;
   }
 
   SDValue N = SDValue(Result, 0);
   SDValue CastRes = DAG.getNode(ISD::BITCAST, DL, CastVT, N);
 
   // Iterate over the result
   SmallVector<SDValue, 4> BVElts;
if (ReqRetNumElts > 1) {
     if (IsD16 && Unpacked)
       PreTFCRes = adjustLoadValueTypeImpl(NewVec, ReqRetVT, DL, DAG, Unpacked);
     else
       PreTFCRes = NewVec;
   } else {
     PreTFCRes = BVElts[0];
   }
 
-  if (V8F16Special)
+  if (V6F16Special)
     PreTFCRes = DAG.getNode(ISD::BITCAST, DL, MVT::v4f16, PreTFCRes);
 
   if (!IsTexFail) {
     if (Result->getNumValues() > 1)
       return DAG.getMergeValues({PreTFCRes, SDValue(Result, 1)}, DL);
     else
       return PreTFCRes;
   }
if (AdjustRetType) {
     if (DMaskLanes == 0 && !BaseOpcode->Store) {
       // This is a no-op load. This can be eliminated
       SDValue Undef = DAG.getUNDEF(Op.getValueType());
       if (isa<MemSDNode>(Op))
         return DAG.getMergeValues({Undef, Op.getOperand(0)}, DL);
       return Undef;
     }
 
-    // Have to use a power of 2 number of dwords
-    NumVDataDwords = 1 << Log2_32_Ceil(NumVDataDwords);
-
     EVT NewVT = NumVDataDwords > 1 ?
                   EVT::getVectorVT(*DAG.getContext(), MVT::f32, NumVDataDwords)
                 : MVT::f32;
 
     ResultTypes[0] = NewVT;
     if (ResultTypes.size() == 3) {
       // Original result was aggregate type used for TexFailCtrl results
       // The actual instruction returns as a vector type which has now been

test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll

 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; NOPRT: v_mov_b32_e32 v4, 0
 ; NOPRT-NOT: v_mov_b32_e32 v0
 ; NOPRT-NOT: v_mov_b32_e32 v1
 ; NOPRT-NOT: v_mov_b32_e32 v2
 ; NOPRT-NOT: v_mov_b32_e32 v3
-; GCN: image_load v[0:7], v{{[0-9]+}}, s[0:7] dmask:0xf unorm tfe{{$}}
+; GCN: image_load v[0:4], v{{[0-9]+}}, s[0:7] dmask:0xf unorm tfe{{$}}
 ; SIVI: buffer_store_dword v4, off, s[8:11], 0
 ; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
 define amdgpu_ps <4 x float> @load_1d_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
   %v.err = extractvalue {<4 x float>, i32} %v, 1
   store i32 %v.err, i32 addrspace(1)* %out, align 4
   ret <4 x float> %v.vec
 }
 
 ; GCN-LABEL: {{^}}load_1d_lwe:
 ; PRT: v_mov_b32_e32 v0, 0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; NOPRT: v_mov_b32_e32 v4, 0
 ; NOPRT-NOT: v_mov_b32_e32 v0
 ; NOPRT-NOT: v_mov_b32_e32 v1
 ; NOPRT-NOT: v_mov_b32_e32 v2
 ; NOPRT-NOT: v_mov_b32_e32 v3
-; GCN: image_load v[0:7], v{{[0-9]+}}, s[0:7] dmask:0xf unorm lwe{{$}}
+; GCN: image_load v[0:4], v{{[0-9]+}}, s[0:7] dmask:0xf unorm lwe{{$}}
 ; SIVI: buffer_store_dword v4, off, s[8:11], 0
 ; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
 define amdgpu_ps <4 x float> @load_1d_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s) {
 main_body:
   %v = call {<4 x float>, i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 2, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
   %v.err = extractvalue {<4 x float>, i32} %v, 1
   store i32 %v.err, i32 addrspace(1)* %out, align 4

 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; NOPRT: v_mov_b32_e32 v4, 0
 ; NOPRT-NOT: v_mov_b32_e32 v0
 ; NOPRT-NOT: v_mov_b32_e32 v1
 ; NOPRT-NOT: v_mov_b32_e32 v2
 ; NOPRT-NOT: v_mov_b32_e32 v3
-; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe{{$}}
+; GCN: image_load v[0:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe{{$}}
 ; SIVI: buffer_store_dword v4, off, s[8:11], 0
 ; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
 define amdgpu_ps <4 x float> @load_2d_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2d.v4f32i32.i32(i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
   %v.err = extractvalue {<4 x float>, i32} %v, 1
   store i32 %v.err, i32 addrspace(1)* %out, align 4

 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; NOPRT: v_mov_b32_e32 v4, 0
 ; NOPRT-NOT: v_mov_b32_e32 v0
 ; NOPRT-NOT: v_mov_b32_e32 v1
 ; NOPRT-NOT: v_mov_b32_e32 v2
 ; NOPRT-NOT: v_mov_b32_e32 v3
-; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe lwe{{$}}
+; GCN: image_load v[0:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe lwe{{$}}
 ; SIVI: buffer_store_dword v4, off, s[8:11], 0
 ; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
 define amdgpu_ps <4 x float> @load_3d_tfe_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %r) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.3d.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 3, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
   %v.err = extractvalue {<4 x float>, i32} %v, 1
   store i32 %v.err, i32 addrspace(1)* %out, align 4

 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; NOPRT: v_mov_b32_e32 v4, 0
 ; NOPRT-NOT: v_mov_b32_e32 v0
 ; NOPRT-NOT: v_mov_b32_e32 v1
 ; NOPRT-NOT: v_mov_b32_e32 v2
 ; NOPRT-NOT: v_mov_b32_e32 v3
-; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe da{{$}}
+; GCN: image_load v[0:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe da{{$}}
 ; SIVI: buffer_store_dword v4, off, s[8:11], 0
 ; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
 define amdgpu_ps <4 x float> @load_cube_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %slice) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.cube.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 2, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
   %v.err = extractvalue {<4 x float>, i32} %v, 1
   store i32 %v.err, i32 addrspace(1)* %out, align 4

 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; NOPRT: v_mov_b32_e32 v4, 0
 ; NOPRT-NOT: v_mov_b32_e32 v0
 ; NOPRT-NOT: v_mov_b32_e32 v1
 ; NOPRT-NOT: v_mov_b32_e32 v2
 ; NOPRT-NOT: v_mov_b32_e32 v3
-; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe da{{$}}
+; GCN: image_load v[0:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe da{{$}}
 ; SIVI: buffer_store_dword v4, off, s[8:11], 0
 ; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
 define amdgpu_ps <4 x float> @load_1darray_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %slice) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1darray.v4f32i32.i32(i32 15, i32 %s, i32 %slice, <8 x i32> %rsrc, i32 1, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
   %v.err = extractvalue {<4 x float>, i32} %v, 1
   store i32 %v.err, i32 addrspace(1)* %out, align 4

 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; NOPRT: v_mov_b32_e32 v4, 0
 ; NOPRT-NOT: v_mov_b32_e32 v0
 ; NOPRT-NOT: v_mov_b32_e32 v1
 ; NOPRT-NOT: v_mov_b32_e32 v2
 ; NOPRT-NOT: v_mov_b32_e32 v3
-; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe da{{$}}
+; GCN: image_load v[0:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe da{{$}}
 ; SIVI: buffer_store_dword v4, off, s[8:11], 0
 ; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
 define amdgpu_ps <4 x float> @load_2darray_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %slice) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2darray.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 2, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
   %v.err = extractvalue {<4 x float>, i32} %v, 1
   store i32 %v.err, i32 addrspace(1)* %out, align 4

 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; NOPRT: v_mov_b32_e32 v4, 0
 ; NOPRT-NOT: v_mov_b32_e32 v0
 ; NOPRT-NOT: v_mov_b32_e32 v1
 ; NOPRT-NOT: v_mov_b32_e32 v2
 ; NOPRT-NOT: v_mov_b32_e32 v3
-; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe lwe{{$}}
+; GCN: image_load v[0:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe lwe{{$}}
 ; SIVI: buffer_store_dword v4, off, s[8:11], 0
 ; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
 define amdgpu_ps <4 x float> @load_2dmsaa_both(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %fragid) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2dmsaa.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 3, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
   %v.err = extractvalue {<4 x float>, i32} %v, 1
   store i32 %v.err, i32 addrspace(1)* %out, align 4

 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; NOPRT: v_mov_b32_e32 v4, 0
 ; NOPRT-NOT: v_mov_b32_e32 v0
 ; NOPRT-NOT: v_mov_b32_e32 v1
 ; NOPRT-NOT: v_mov_b32_e32 v2
 ; NOPRT-NOT: v_mov_b32_e32 v3
-; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe da{{$}}
+; GCN: image_load v[0:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe da{{$}}
 ; SIVI: buffer_store_dword v4, off, s[8:11], 0
 ; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
 define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2darraymsaa.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
   %v.err = extractvalue {<4 x float>, i32} %v, 1
   store i32 %v.err, i32 addrspace(1)* %out, align 4

 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; NOPRT: v_mov_b32_e32 v4, 0
 ; NOPRT-NOT: v_mov_b32_e32 v0
 ; NOPRT-NOT: v_mov_b32_e32 v1
 ; NOPRT-NOT: v_mov_b32_e32 v2
 ; NOPRT-NOT: v_mov_b32_e32 v3
-; GCN: image_load_mip v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe{{$}}
+; GCN: image_load_mip v[0:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe{{$}}
 ; SIVI: buffer_store_dword v4, off, s[8:11], 0
 ; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
 define amdgpu_ps <4 x float> @load_mip_1d_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %mip) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.mip.1d.v4f32i32.i32(i32 15, i32 %s, i32 %mip, <8 x i32> %rsrc, i32 2, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
   %v.err = extractvalue {<4 x float>, i32} %v, 1
   store i32 %v.err, i32 addrspace(1)* %out, align 4

 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; NOPRT: v_mov_b32_e32 v4, 0
 ; NOPRT-NOT: v_mov_b32_e32 v0
 ; NOPRT-NOT: v_mov_b32_e32 v1
 ; NOPRT-NOT: v_mov_b32_e32 v2
 ; NOPRT-NOT: v_mov_b32_e32 v3
-; GCN: image_load_mip v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe{{$}}
+; GCN: image_load_mip v[0:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe{{$}}
 ; SIVI: buffer_store_dword v4, off, s[8:11], 0
 ; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
 define amdgpu_ps <4 x float> @load_mip_2d_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %mip) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.mip.2d.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %mip, <8 x i32> %rsrc, i32 1, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
   %v.err = extractvalue {<4 x float>, i32} %v, 1
   store i32 %v.err, i32 addrspace(1)* %out, align 4

 
 ; GCN-LABEL: {{^}}load_1d_tfe_V4_dmask2:
 ; PRT: v_mov_b32_e32 v0, 0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
 ; NOPRT: v_mov_b32_e32 v2, 0
 ; NOPRT-NOT: v_mov_b32_e32 v0
 ; NOPRT-NOT: v_mov_b32_e32 v1
-; GCN: image_load v[0:3], v{{[0-9]+}}, s[0:7] dmask:0x6 unorm tfe{{$}}
+; GCN: image_load v[0:2], v{{[0-9]+}}, s[0:7] dmask:0x6 unorm tfe{{$}}
 ; SIVI: buffer_store_dword v2, off, s[8:11], 0
 ; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v2
 define amdgpu_ps <4 x float> @load_1d_tfe_V4_dmask2(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 6, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
   %v.err = extractvalue {<4 x float>, i32} %v, 1
   store i32 %v.err, i32 addrspace(1)* %out, align 4

test/CodeGen/AMDGPU/llvm.amdgcn.image.load.a16.ll

 define amdgpu_ps <4 x float> @load.v2f32.1d(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
 main_body:
   %x = extractelement <2 x i16> %coords, i32 0
   %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32 3, i16 %x, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 ; GCN-LABEL: {{^}}load.v3f32.1d:
-; GCN: image_load v[0:3], v0, s[0:7] dmask:0x7 unorm a16
+; GCN: image_load v[0:2], v0, s[0:7] dmask:0x7 unorm a16
 define amdgpu_ps <4 x float> @load.v3f32.1d(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
 main_body:
   %x = extractelement <2 x i16> %coords, i32 0
   %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32 7, i16 %x, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 ; GCN-LABEL: {{^}}load.v4f32.1d:

 main_body:
   %x = extractelement <2 x i16> %coords, i32 0
   %y = extractelement <2 x i16> %coords, i32 1
   %v = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i16(i32 3, i16 %x, i16 %y, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 ; GCN-LABEL: {{^}}load.v3f32.2d:
-; GCN: image_load v[0:3], v0, s[0:7] dmask:0x7 unorm a16
+; GCN: image_load v[0:2], v0, s[0:7] dmask:0x7 unorm a16
 define amdgpu_ps <4 x float> @load.v3f32.2d(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
 main_body:
   %x = extractelement <2 x i16> %coords, i32 0
   %y = extractelement <2 x i16> %coords, i32 1
   %v = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i16(i32 7, i16 %x, i16 %y, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
main_body:
   %x = extractelement <2 x i16> %coords_lo, i32 0
   %y = extractelement <2 x i16> %coords_lo, i32 1
   %z = extractelement <2 x i16> %coords_hi, i32 0
   %v = call <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i16(i32 3, i16 %x, i16 %y, i16 %z, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 ; GCN-LABEL: {{^}}load.v3f32.3d:
-; GCN: image_load v[0:3], v[0:1], s[0:7] dmask:0x7 unorm a16
+; GCN: image_load v[0:2], v[0:1], s[0:7] dmask:0x7 unorm a16
 define amdgpu_ps <4 x float> @load.v3f32.3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_lo, <2 x i16> %coords_hi) {
 main_body:
   %x = extractelement <2 x i16> %coords_lo, i32 0
   %y = extractelement <2 x i16> %coords_lo, i32 1
   %z = extractelement <2 x i16> %coords_hi, i32 0
   %v = call <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i16(i32 7, i16 %x, i16 %y, i16 %z, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }

test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll

 }
 
 ; GCN-LABEL: {{^}}sample_1d_tfe:
 ; GCN: v_mov_b32_e32 v0, 0
 ; GCN: v_mov_b32_e32 v1, v0
 ; GCN: v_mov_b32_e32 v2, v0
 ; GCN: v_mov_b32_e32 v3, v0
 ; GCN: v_mov_b32_e32 v4, v0
-; GCN: image_sample v[0:7], v5, s[0:7], s[8:11] dmask:0xf tfe{{$}}
+; GCN: image_sample v[0:4], v5, s[0:7], s[8:11] dmask:0xf tfe{{$}}
 define amdgpu_ps <4 x float> @sample_1d_tfe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 addrspace(1)* inreg %out, float %s) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
   %v.err = extractvalue {<4 x float>, i32} %v, 1
   store i32 %v.err, i32 addrspace(1)* %out, align 4
   ret <4 x float> %v.vec
 }

 }
 
 ; GCN-LABEL: {{^}}sample_1d_lwe:
 ; GCN: v_mov_b32_e32 v0, 0
 ; GCN: v_mov_b32_e32 v1, v0
 ; GCN: v_mov_b32_e32 v2, v0
 ; GCN: v_mov_b32_e32 v3, v0
 ; GCN: v_mov_b32_e32 v4, v0
-; GCN: image_sample v[0:7], v5, s[0:7], s[8:11] dmask:0xf lwe{{$}}
+; GCN: image_sample v[0:4], v5, s[0:7], s[8:11] dmask:0xf lwe{{$}}
 define amdgpu_ps <4 x float> @sample_1d_lwe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 addrspace(1)* inreg %out, float %s) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 2, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
   %v.err = extractvalue {<4 x float>, i32} %v, 1
   store i32 %v.err, i32 addrspace(1)* %out, align 4
   ret <4 x float> %v.vec
 }

 ; GCN: image_sample_c_d_o v[0:1], v[0:15], s[0:7], s[8:11] dmask:0x6 da{{$}}
 define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %slice) {
 main_body:
   %v = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f32.f32(i32 6, i32 %offset, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   ret <2 x float> %v
 }
 
 ; GCN-LABEL: {{^}}sample_c_d_o_2darray_V2_tfe:
-; GCN: image_sample_c_d_o v[9:12], v[0:15], s[0:7], s[8:11] dmask:0x6 tfe da{{$}}
+; GCN: image_sample_c_d_o v[9:11], v[0:15], s[0:7], s[8:11] dmask:0x6 tfe da{{$}}
 define amdgpu_ps <4 x float> @sample_c_d_o_2darray_V2_tfe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %slice) {
 main_body:
   %v = call {<2 x float>, i32} @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32i32.f32.f32(i32 6, i32 %offset, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0)
   %v.vec = extractvalue {<2 x float>, i32} %v, 0
   %v.f1 = extractelement <2 x float> %v.vec, i32 0
   %v.f2 = extractelement <2 x float> %v.vec, i32 1
   %v.err = extractvalue {<2 x float>, i32} %v, 1
   %v.errf = bitcast i32 %v.err to float