DAG: Handle odd vector sizes in calling conv splitting

[arsenm]

Summary

This already worked if only one register piece was used,
but didn't if a type was split into multiple, unequal
sized pieces.

Fixes not splitting 3i16/v3f16 into two registers for
AMDGPU.

This will also allow fixing the ABI for 16-bit vectors
in a future commit so that it's the same for all subtargets.

[arsenm]

ping

[arsenm]

ping

[nhaehnle]

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

728–731

If I understand correctly what the code is trying to do, i * IntermediateNumElts would also work, right? That would be much clearer IMHO.

Also, this definition can be moved into the if case.

[RKSimon]

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

618

Please could you pull this helper out in a NFCI patch first? Then any functional changes you've made will be much easier to spot.

[arsenm]

Split out piece, change index calculation

[arsenm]

ping

[RKSimon]

lib/Target/AMDGPU/SIISelLowering.cpp

741–742

Update the FIXME comment?

[arsenm]

Fix comment

[nhaehnle]

LGTM

[arsenm]

r341801

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
     }
   }
 
   if (DAG.getDataLayout().isBigEndian())
     std::reverse(Parts, Parts + OrigNumParts);
 }
 
 static SDValue widenVectorToPartType(SelectionDAG &DAG,
                                      SDValue Val, const SDLoc &DL, EVT PartVT) {
   if (!PartVT.isVector())
     return SDValue();
 
   EVT ValueVT = Val.getValueType();
   unsigned PartNumElts = PartVT.getVectorNumElements();
   unsigned ValueNumElts = ValueVT.getVectorNumElements();
   if (PartNumElts > ValueNumElts &&
       PartVT.getVectorElementType() == ValueVT.getVectorElementType()) {
if (IsABIRegCopy) {
     NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
         *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
         NumIntermediates, RegisterVT);
   } else {
     NumRegs =
         TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
                                    NumIntermediates, RegisterVT);
   }
-  unsigned NumElements = ValueVT.getVectorNumElements();
 
   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
   NumParts = NumRegs; // Silence a compiler warning.
   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
 
+  unsigned IntermediateNumElts = IntermediateVT.isVector() ?
+    IntermediateVT.getVectorNumElements() : 1;
+
   // Convert the vector to the appropiate type if necessary.
-  unsigned DestVectorNoElts =
-      NumIntermediates *
-      (IntermediateVT.isVector() ? IntermediateVT.getVectorNumElements() : 1);
+  unsigned DestVectorNoElts = NumIntermediates * IntermediateNumElts;
+
   EVT BuiltVectorTy = EVT::getVectorVT(
       *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts);
-  if (Val.getValueType() != BuiltVectorTy)
+  MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
+  if (ValueVT != BuiltVectorTy) {
+    if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))
+      Val = Widened;
+
     Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
+  }
 
   // Split the vector into intermediate operands.
   SmallVector<SDValue, 8> Ops(NumIntermediates);
   for (unsigned i = 0; i != NumIntermediates; ++i) {
-    if (IntermediateVT.isVector())
-      Ops[i] =
-          DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
-                      DAG.getConstant(i * (NumElements / NumIntermediates), DL,
-                                      TLI.getVectorIdxTy(DAG.getDataLayout())));
-    else
+    if (IntermediateVT.isVector()) {
+      Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
+                           DAG.getConstant(i * IntermediateNumElts, DL, IdxVT));
+    } else {
       Ops[i] = DAG.getNode(
           ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
-          DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
+          DAG.getConstant(i, DL, IdxVT));
+    }
   }
 
   // Split the intermediate operands into legal parts.
   if (NumParts == NumIntermediates) {
     // If the register was not expanded, promote or copy the value,
     // as appropriate.
     for (unsigned i = 0; i != NumParts; ++i)
       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);

lib/Target/AMDGPU/SIISelLowering.cpp

if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) {
     EVT ScalarVT = VT.getScalarType();
     unsigned Size = ScalarVT.getSizeInBits();
     if (Size == 32)
       return ScalarVT.getSimpleVT();
 
     if (Size == 64)
       return MVT::i32;
 
-    if (Size == 16 &&
-        Subtarget->has16BitInsts() &&
-        isPowerOf2_32(VT.getVectorNumElements()))
+    if (Size == 16 && Subtarget->has16BitInsts())
       return VT.isInteger() ? MVT::v2i16 : MVT::v2f16;
   }
 
   return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
 }
 
 unsigned SITargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
                                                          CallingConv::ID CC,
                                                          EVT VT) const {
   if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) {
     unsigned NumElts = VT.getVectorNumElements();
     EVT ScalarVT = VT.getScalarType();
     unsigned Size = ScalarVT.getSizeInBits();
 
     if (Size == 32)
       return NumElts;
 
     if (Size == 64)
       return 2 * NumElts;
 
-    // FIXME: Fails to break down as we want with v3.
-    if (Size == 16 && Subtarget->has16BitInsts() && isPowerOf2_32(NumElts))
-      return VT.getVectorNumElements() / 2;
+    if (Size == 16 && Subtarget->has16BitInsts())
+      return (VT.getVectorNumElements() + 1) / 2;
   }
 
   return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
 }
 
 unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv(
   LLVMContext &Context, CallingConv::ID CC,
   EVT VT, EVT &IntermediateVT,
if (Size == 64) {
       IntermediateVT = RegisterVT;
       NumIntermediates = 2 * NumElts;
       return NumIntermediates;
     }
 
     // FIXME: We should fix the ABI to be the same on targets without 16-bit
     // support, but unless we can properly handle 3-vectors, it will be still be
     // inconsistent.
-    if (Size == 16 && Subtarget->has16BitInsts() && isPowerOf2_32(NumElts)) {
+    if (Size == 16 && Subtarget->has16BitInsts()) {
       RegisterVT = VT.isInteger() ? MVT::v2i16 : MVT::v2f16;
       IntermediateVT = RegisterVT;
-      NumIntermediates = NumElts / 2;
+      NumIntermediates = (NumElts + 1) / 2;
       return NumIntermediates;
     }
   }
 
   return TargetLowering::getVectorTypeBreakdownForCallingConv(
     Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
 }
 

test/CodeGen/AMDGPU/call-argument-types.ll

 ; GFX9-NOT: v1
 ; GFX9: s_swappc_b64
 define amdgpu_kernel void @test_call_external_void_func_v3i16() #0 {
   %val = load <3 x i16>, <3 x i16> addrspace(1)* undef
   call void @external_void_func_v3i16(<3 x i16> %val)
   ret void
 }
 
-; FIXME: materialize constant directly in VGPR
+; GCN-LABEL: {{^}}test_call_external_void_func_v3f16:
+; GFX9: buffer_load_dwordx2 v[0:1]
+; GFX9-NOT: v0
+; GFX9-NOT: v1
+; GFX9: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_v3f16() #0 {
+  %val = load <3 x half>, <3 x half> addrspace(1)* undef
+  call void @external_void_func_v3f16(<3 x half> %val)
+  ret void
+}
+
 ; GCN-LABEL: {{^}}test_call_external_void_func_v3i16_imm:
-; GFX9-DAG: s_mov_b32 [[K01:s[0-9]+]], 0x20001
-; GFX9-DAG: s_mov_b32 [[K2:s[0-9]+]], 3
-; GFX9: v_mov_b32_e32 v0, [[K01]]
-; GFX9: v_mov_b32_e32 v1, [[K2]]
+; GFX9: v_mov_b32_e32 v0, 0x20001
+; GFX9: v_mov_b32_e32 v1, 3
 ; GFX9: s_swappc_b64
 define amdgpu_kernel void @test_call_external_void_func_v3i16_imm() #0 {
   call void @external_void_func_v3i16(<3 x i16> <i16 1, i16 2, i16 3>)
   ret void
 }
 
+; GCN-LABEL: {{^}}test_call_external_void_func_v3f16_imm:
+; GFX9: v_mov_b32_e32 v0, 0x40003c00
+; GFX9: v_mov_b32_e32 v1, 0x4400
+; GFX9: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_v3f16_imm() #0 {
+  call void @external_void_func_v3f16(<3 x half> <half 1.0, half 2.0, half 4.0>)
+  ret void
+}
+
 ; GCN-LABEL: {{^}}test_call_external_void_func_v4i16:
 ; GFX9: buffer_load_dwordx2 v[0:1]
 ; GFX9-NOT: v0
 ; GFX9-NOT: v1
 ; GFX9: s_swappc_b64
 define amdgpu_kernel void @test_call_external_void_func_v4i16() #0 {
   %val = load <4 x i16>, <4 x i16> addrspace(1)* undef
   call void @external_void_func_v4i16(<4 x i16> %val)

test/CodeGen/AMDGPU/fcanonicalize.f16.ll

define amdgpu_kernel void @test_fold_canonicalize_snan3_value_v2f16(<2 x half> addrspace(1)* %out) #1 {
   %canonicalized = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> <half 0xHFC01, half 0xHFC01>)
   store <2 x half> %canonicalized, <2 x half> addrspace(1)* %out
   ret void
 }
 
 ; FIXME: Extra 4th component handled
 ; GCN-LABEL: {{^}}v_test_canonicalize_var_v3f16:
 ; GFX9: s_waitcnt
-; GFX9-NEXT: v_pk_max_f16 v1, v1, v1
 ; GFX9-NEXT: v_pk_max_f16 v0, v0, v0
+; GFX9-NEXT: v_pk_max_f16 v1, v1, v1
 ; GFX9-NEXT: s_setpc_b64
 
-; VI-DAG: v_max_f16_sdwa [[CANON_ELT3:v[0-9]+]], v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_max_f16_e32 [[CANON_ELT2:v[0-9]+]], v1, v1
 ; VI-DAG: v_max_f16_sdwa [[CANON_ELT1:v[0-9]+]], v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI-DAG: v_max_f16_e32 [[CANON_ELT0:v[0-9]+]], v0, v0
+; VI-DAG: v_max_f16_e32 v1, v1, v1
 ; VI-DAG: v_or_b32_e32 v0, [[CANON_ELT0]], [[CANON_ELT1]]
-; VI-DAG: v_or_b32_e32 v1, [[CANON_ELT2]], [[CANON_ELT3]]
+
 ; VI: s_setpc_b64
 define <3 x half> @v_test_canonicalize_var_v3f16(<3 x half> %val) #1 {
   %canonicalized = call <3 x half> @llvm.canonicalize.v3f16(<3 x half> %val)
   ret <3 x half> %canonicalized
 }
 
 ; GCN-LABEL: {{^}}v_test_canonicalize_var_v4f16:
 ; GFX9: s_waitcnt

test/CodeGen/AMDGPU/fmax_legacy.f16.ll

 ; GFX9-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX9-SAFE-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-SAFE-NEXT:    v_lshl_or_b32 v0, v4, 16, v0
 ; GFX9-SAFE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-NNAN-LABEL: test_fmax_legacy_ugt_v3f16:
 ; GFX9-NNAN:       ; %bb.0:
 ; GFX9-NNAN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NNAN-NEXT:    v_pk_max_f16 v1, v1, v3
 ; GFX9-NNAN-NEXT:    v_pk_max_f16 v0, v0, v2
+; GFX9-NNAN-NEXT:    v_pk_max_f16 v1, v1, v3
 ; GFX9-NNAN-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-SAFE-LABEL: test_fmax_legacy_ugt_v3f16:
 ; VI-SAFE:       ; %bb.0:
 ; VI-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SAFE-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
 ; VI-SAFE-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
 ; VI-SAFE-NEXT:    v_cmp_nle_f16_e32 vcc, v5, v4

test/CodeGen/AMDGPU/fmin_legacy.f16.ll

 ; GFX9-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX9-SAFE-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-SAFE-NEXT:    v_lshl_or_b32 v0, v4, 16, v0
 ; GFX9-SAFE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-NNAN-LABEL: test_fmin_legacy_ule_v3f16:
 ; GFX9-NNAN:       ; %bb.0:
 ; GFX9-NNAN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NNAN-NEXT:    v_pk_min_f16 v1, v1, v3
 ; GFX9-NNAN-NEXT:    v_pk_min_f16 v0, v0, v2
+; GFX9-NNAN-NEXT:    v_pk_min_f16 v1, v1, v3
 ; GFX9-NNAN-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-SAFE-LABEL: test_fmin_legacy_ule_v3f16:
 ; VI-SAFE:       ; %bb.0:
 ; VI-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SAFE-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
 ; VI-SAFE-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
 ; VI-SAFE-NEXT:    v_cmp_ngt_f16_e32 vcc, v5, v4

test/CodeGen/AMDGPU/function-args.ll

 ; GFX9-NOT: v1
 ; GFX9: buffer_store_dwordx2 v[0:1], off
 define void @void_func_v4i16(<4 x i16> %arg0) #0 {
   store <4 x i16> %arg0, <4 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}void_func_v5i16:
-; GCN-DAG: buffer_store_short v4, off,
-; GCN-DAG: buffer_store_dwordx2 v[1:2], off
+; CI: v_lshlrev_b32
+; CI: v_and_b32
+; CI: v_lshlrev_b32
+; CI: v_or_b32
+; CI: v_or_b32
+; CI-DAG: buffer_store_short v
+; CI-DAG: buffer_store_dwordx2 v
+
+; GFX89-DAG: buffer_store_short v2, off,
+; GFX89-DAG: buffer_store_dwordx2 v[0:1], off
+
 define void @void_func_v5i16(<5 x i16> %arg0) #0 {
   store <5 x i16> %arg0, <5 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}void_func_v8i16:
 ; GFX9-DAG: buffer_store_dwordx4 v[0:3], off
 define void @void_func_v8i16(<8 x i16> %arg0) #0 {

test/CodeGen/AMDGPU/function-returns.ll

 ; GFX9: buffer_load_dwordx2 v[0:1], off
 ; GFX9-NEXT: s_waitcnt vmcnt(0)
 ; GFX9-NEXT: s_setpc_b64
 define <4 x half> @v4f16_func_void() #0 {
   %val = load <4 x half>, <4 x half> addrspace(1)* undef
   ret <4 x half> %val
 }
 
+; FIXME: Mixing buffer and global
 ; FIXME: Should not scalarize
 ; GCN-LABEL: {{^}}v5i16_func_void:
 ; GFX9: buffer_load_dwordx2 v[0:1]
-; GFX9: buffer_load_ushort v4
-; GFX9: v_lshrrev_b32_e32 v5, 16, v0
-; GFX9: v_lshrrev_b32_e32 v3, 16, v1
-; GCN: s_setpc_b64
+; GFX9-NEXT: global_load_short_d16 v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
 define <5 x i16> @v5i16_func_void() #0 {
   %ptr = load volatile <5 x i16> addrspace(1)*, <5 x i16> addrspace(1)* addrspace(4)* undef
   %val = load <5 x i16>, <5 x i16> addrspace(1)* %ptr
   ret <5 x i16> %val
 }
 
 ; GCN-LABEL: {{^}}v8i16_func_void:
 ; GFX9-DAG: buffer_load_dwordx4 v[0:3], off

test/CodeGen/AMDGPU/mad-mix-lo.ll

define <2 x half> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
   %src2.ext = fpext <2 x half> %src2 to <2 x float>
   %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
   %cvt.result = fptrunc <2 x float> %result to <2 x half>
   ret <2 x half> %cvt.result
 }
 
 ; GCN-LABEL: {{^}}v_mad_mix_v3f32:
 ; GCN: s_waitcnt
-; GFX9-NEXT: v_mad_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
-; GFX9-NEXT: v_mad_mixlo_f16 v7, v1, v3, v5 op_sel_hi:[1,1,1]
-; GFX9-NEXT: v_mad_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; GFX9-NEXT: v_mad_mixhi_f16 v7, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; GFX9-NEXT: v_mov_b32_e32 v0, v6
-; GFX9-NEXT: v_mov_b32_e32 v1, v7
+; GFX9-NEXT: v_mad_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
+; GFX9-NEXT: v_mad_mixlo_f16 v3, v0, v2, v4 op_sel_hi:[1,1,1]
+; GFX9-NEXT: v_mad_mixhi_f16 v3, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GFX9-NEXT: v_mov_b32_e32 v0, v3
 ; GFX9-NEXT: s_setpc_b64
 define <3 x half> @v_mad_mix_v3f32(<3 x half> %src0, <3 x half> %src1, <3 x half> %src2) #0 {
   %src0.ext = fpext <3 x half> %src0 to <3 x float>
   %src1.ext = fpext <3 x half> %src1 to <3 x float>
   %src2.ext = fpext <3 x half> %src2 to <3 x float>
   %result = tail call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %src0.ext, <3 x float> %src1.ext, <3 x float> %src2.ext)
   %cvt.result = fptrunc <3 x float> %result to <3 x half>
   ret <3 x half> %cvt.result
define <2 x half> @v_mad_mix_v2f32_clamp_postcvt(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
   %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
   ret <2 x half> %clamp
 }
 
 ; FIXME: Should be packed into 2 registers per argument?
 ; GCN-LABEL: {{^}}v_mad_mix_v3f32_clamp_postcvt:
 ; GCN: s_waitcnt
 ; GFX9-NEXT: v_mad_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] clamp
-; GFX9-NEXT: v_mad_mixlo_f16 v7, v1, v3, v5 op_sel_hi:[1,1,1] clamp
 ; GFX9-NEXT: v_mad_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
-; GFX9-NEXT: v_mad_mixhi_f16 v7, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; GFX9-NEXT: v_mad_mixlo_f16 v2, v1, v3, v5 op_sel_hi:[1,1,1] clamp
+; GFX9-NEXT: v_mad_mixhi_f16 v2, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
 ; GFX9-NEXT: v_mov_b32_e32 v0, v6
-; GFX9-NEXT: v_mov_b32_e32 v1, v7
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
 ; GFX9-NEXT: s_setpc_b64
 define <3 x half> @v_mad_mix_v3f32_clamp_postcvt(<3 x half> %src0, <3 x half> %src1, <3 x half> %src2) #0 {
   %src0.ext = fpext <3 x half> %src0 to <3 x float>
   %src1.ext = fpext <3 x half> %src1 to <3 x float>
   %src2.ext = fpext <3 x half> %src2 to <3 x float>
   %result = tail call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %src0.ext, <3 x float> %src1.ext, <3 x float> %src2.ext)
   %cvt.result = fptrunc <3 x float> %result to <3 x half>
   %max = call <3 x half> @llvm.maxnum.v3f16(<3 x half> %cvt.result, <3 x half> zeroinitializer)
define <2 x half> @v_mad_mix_v2f32_clamp_precvt(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
   %max = call <2 x float> @llvm.maxnum.v2f32(<2 x float> %result, <2 x float> zeroinitializer)
   %clamp = call <2 x float> @llvm.minnum.v2f32(<2 x float> %max, <2 x float> <float 1.0, float 1.0>)
   %cvt.result = fptrunc <2 x float> %clamp to <2 x half>
   ret <2 x half> %cvt.result
 }
 
 ; FIXME: Handling undef 4th component
 ; GCN-LABEL: {{^}}v_mad_mix_v3f32_clamp_precvt:
-; GFX9: v_mad_mix_f32 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
-; GFX9: v_mad_mix_f32 v0, v0, v2, v4 op_sel_hi:[1,1,1] clamp
-; GFX9: v_mad_mix_f32 v2, v1, v3, v5 op_sel_hi:[1,1,1] clamp
-; GFX9: v_mad_mix_f32 v1, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-
-; GFX9: v_cvt_f16_f32
-; GFX9: v_cvt_f16_f32
-; GFX9: v_cvt_f16_f32
-; GFX9: v_cvt_f16_f32
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v2, v4 op_sel_hi:[1,1,1] clamp
+; GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX9-NEXT: v_mad_mix_f32 v1, v1, v3, v5 op_sel_hi:[1,1,1] clamp
+; GFX9-NEXT: v_cvt_f16_f32_e32 v2, v6
+; GFX9-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: s_setpc_b64
 define <3 x half> @v_mad_mix_v3f32_clamp_precvt(<3 x half> %src0, <3 x half> %src1, <3 x half> %src2) #0 {
   %src0.ext = fpext <3 x half> %src0 to <3 x float>
   %src1.ext = fpext <3 x half> %src1 to <3 x float>
   %src2.ext = fpext <3 x half> %src2 to <3 x float>
   %result = tail call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %src0.ext, <3 x float> %src1.ext, <3 x float> %src2.ext)
   %max = call <3 x float> @llvm.maxnum.v3f32(<3 x float> %result, <3 x float> zeroinitializer)
   %clamp = call <3 x float> @llvm.minnum.v3f32(<3 x float> %max, <3 x float> <float 1.0, float 1.0, float 1.0>)
   %cvt.result = fptrunc <3 x float> %clamp to <3 x half>