SelectionDAG: Match min/max if the scalar operation is legal

[hfinkel]

Can you please explain the logic behind this change?

[arsenm]

In D14038#276523, @hfinkel wrote:

Can you please explain the logic behind this change?

This min/max pattern matching is weird because it's done in SelectionDAGBuilder. It won't match for any vector if the vector operation isn't legal. AMDGPU only has vector loads and stores, and every operation is scalar, so we only care about the legality of the scalar operations.

[arsenm]

ping

[hfinkel]

In D14038#276589, @arsenm wrote:

In D14038#276523, @hfinkel wrote:

Can you please explain the logic behind this change?

This min/max pattern matching is weird because it's done in SelectionDAGBuilder. It won't match for any vector if the vector operation isn't legal. AMDGPU only has vector loads and stores, and every operation is scalar, so we only care about the legality of the scalar operations.

Okay, I think this makes sense: You have legal vector types, but few legal vector operations (only memory accesses). We do MIN/MAX matching during SDAGBuilder, and so if you don't have a vector MIN/MAX operation, but you have vector operations representing a MIN/MAX operation, and you do have a MIN/MAX on the scalar types, then it will be scalarized regardless, and you want it to be scalarized into scalar MINs/MAXs, instead of being scalarized into some collection of other operations.

Please add more-verbose commentary explaining what is going on. Also, I think the implementation needs to deal with promotions in order to work as intended in all cases. The question you're asking is: Is the operation on this type legal, or will it be legal after the operation is legalized. To do this, you need to call getOperationAction, and if you get back Promote, call it again with the promoted-to type. You can get the type by calling getTypeToPromoteTo.

[arsenm]

Change existing loop with getTypeToTransformTo to find final type. Try to use the scalar operation if VSELECT is illegal.

[hfinkel]

LGTM.

[arsenm]

r255388

lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

#endif
   case ISD::FADD:
   case ISD::FCOPYSIGN:
   case ISD::FDIV:
   case ISD::FMUL:
   case ISD::FMINNUM:
   case ISD::FMAXNUM:
   case ISD::FMINNAN:
   case ISD::FMAXNAN:
+  case ISD::SMIN:
+  case ISD::SMAX:
+  case ISD::UMIN:
+  case ISD::UMAX:
 
   case ISD::FPOW:
   case ISD::FREM:
   case ISD::FSUB:
   case ISD::MUL:
   case ISD::OR:
   case ISD::SDIV:
   case ISD::SREM:
#endif
   case ISD::MULHU:
   case ISD::OR:
   case ISD::SUB:
   case ISD::XOR:
   case ISD::FMINNUM:
   case ISD::FMAXNUM:
   case ISD::FMINNAN:
   case ISD::FMAXNAN:
+  case ISD::SMIN:
+  case ISD::SMAX:
+  case ISD::UMIN:
+  case ISD::UMAX:
     Res = WidenVecRes_Binary(N);
     break;
 
   case ISD::FADD:
   case ISD::FMUL:
   case ISD::FPOW:
   case ISD::FSUB:
   case ISD::FDIV:

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

void SelectionDAGBuilder::visitSelect(const User &I) {
   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
     ISD::VSELECT : ISD::SELECT;
 
   // Min/max matching is only viable if all output VTs are the same.
   if (std::equal(ValueVTs.begin(), ValueVTs.end(), ValueVTs.begin())) {
     EVT VT = ValueVTs[0];
     LLVMContext &Ctx = *DAG.getContext();
     auto &TLI = DAG.getTargetLoweringInfo();
-    while (TLI.getTypeAction(Ctx, VT) == TargetLoweringBase::TypeSplitVector)
+
+    // We care about the legality of the operation after it has been type
+    // legalized.
+    while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
       VT = TLI.getTypeToTransformTo(Ctx, VT);
 
+    // If the vselect is legal, assume we want to leave this as a vector setcc +
+    // vselect. Otherwise, if this is going to be scalarized, we want to see if
+    // min/max is legal on the scalar type.
+    bool UseScalarMinMax = VT.isVector() &&
+      !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
+
     Value *LHS, *RHS;
     auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
     ISD::NodeType Opc = ISD::DELETED_NODE;
     switch (SPR.Flavor) {
     case SPF_UMAX:    Opc = ISD::UMAX; break;
     case SPF_UMIN:    Opc = ISD::UMIN; break;
     case SPF_SMAX:    Opc = ISD::SMAX; break;
     case SPF_SMIN:    Opc = ISD::SMIN; break;
     case SPF_FMINNUM:
       switch (SPR.NaNBehavior) {
       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
       case SPNB_RETURNS_NAN:   Opc = ISD::FMINNAN; break;
       case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
-      case SPNB_RETURNS_ANY:
-        Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT) ? ISD::FMINNUM
-          : ISD::FMINNAN;
+      case SPNB_RETURNS_ANY: {
+        if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
+          Opc = ISD::FMINNUM;
+        else if (TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT))
+          Opc = ISD::FMINNAN;
+        else if (UseScalarMinMax)
+          Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
+            ISD::FMINNUM : ISD::FMINNAN;
         break;
       }
+      }
       break;
     case SPF_FMAXNUM:
       switch (SPR.NaNBehavior) {
       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
       case SPNB_RETURNS_NAN:   Opc = ISD::FMAXNAN; break;
       case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
       case SPNB_RETURNS_ANY:
-        Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT) ? ISD::FMAXNUM
-          : ISD::FMAXNAN;
+
+        if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
+          Opc = ISD::FMAXNUM;
+        else if (TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT))
+          Opc = ISD::FMAXNAN;
+        else if (UseScalarMinMax)
+          Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
+            ISD::FMAXNUM : ISD::FMAXNAN;
         break;
       }
       break;
     default: break;
     }
 
-    if (Opc != ISD::DELETED_NODE && TLI.isOperationLegalOrCustom(Opc, VT) &&
-        // If the underlying comparison instruction is used by any other instruction,
-        // the consumed instructions won't be destroyed, so it is not profitable
-        // to convert to a min/max.
+    if (Opc != ISD::DELETED_NODE &&
+        (TLI.isOperationLegalOrCustom(Opc, VT) ||
+         (UseScalarMinMax &&
+          TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
+        // If the underlying comparison instruction is used by any other
+        // instruction, the consumed instructions won't be destroyed, so it is
+        // not profitable to convert to a min/max.
         cast<SelectInst>(&I)->getCondition()->hasOneUse()) {
       OpCode = Opc;
       LHSVal = getValue(LHS);
       RHSVal = getValue(RHS);
       BaseOps = {};
     }
   }
 

test/CodeGen/AMDGPU/fmax_legacy.ll

define void @test_fmax_legacy_ogt_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
   %b = load float, float addrspace(1)* %gep.1, align 4
 
   %cmp = fcmp ogt float %a, %b
   %val = select i1 %cmp, float %a, float %b
   store float %val, float addrspace(1)* %out, align 4
   ret void
 }
 
+; FUNC-LABEL: {{^}}test_fmax_legacy_ogt_v1f32:
+; SI: buffer_load_dword [[A:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; SI: buffer_load_dword [[B:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
+; SI-SAFE: v_max_legacy_f32_e32 {{v[0-9]+}}, [[A]], [[B]]
+; SI-NONAN: v_max_f32_e32 {{v[0-9]+}}, [[B]], [[A]]
+; EG: MAX
+define void @test_fmax_legacy_ogt_v1f32(<1 x float> addrspace(1)* %out, <1 x float> addrspace(1)* %in) #0 {
+  %tid = call i32 @llvm.r600.read.tidig.x() #1
+  %gep.0 = getelementptr <1 x float>, <1 x float> addrspace(1)* %in, i32 %tid
+  %gep.1 = getelementptr <1 x float>, <1 x float> addrspace(1)* %gep.0, i32 1
+
+  %a = load <1 x float>, <1 x float> addrspace(1)* %gep.0
+  %b = load <1 x float>, <1 x float> addrspace(1)* %gep.1
+
+  %cmp = fcmp ogt <1 x float> %a, %b
+  %val = select <1 x i1> %cmp, <1 x float> %a, <1 x float> %b
+  store <1 x float> %val, <1 x float> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_fmax_legacy_ogt_v3f32:
+; SI-SAFE: v_max_legacy_f32_e32
+; SI-SAFE: v_max_legacy_f32_e32
+; SI-SAFE: v_max_legacy_f32_e32
+; SI-NONAN: v_max_f32_e32
+; SI-NONAN: v_max_f32_e32
+; SI-NONAN: v_max_f32_e32
+define void @test_fmax_legacy_ogt_v3f32(<3 x float> addrspace(1)* %out, <3 x float> addrspace(1)* %in) #0 {
+  %tid = call i32 @llvm.r600.read.tidig.x() #1
+  %gep.0 = getelementptr <3 x float>, <3 x float> addrspace(1)* %in, i32 %tid
+  %gep.1 = getelementptr <3 x float>, <3 x float> addrspace(1)* %gep.0, i32 1
+
+  %a = load <3 x float>, <3 x float> addrspace(1)* %gep.0
+  %b = load <3 x float>, <3 x float> addrspace(1)* %gep.1
+
+  %cmp = fcmp ogt <3 x float> %a, %b
+  %val = select <3 x i1> %cmp, <3 x float> %a, <3 x float> %b
+  store <3 x float> %val, <3 x float> addrspace(1)* %out
+  ret void
+}
 
 ; FUNC-LABEL: @test_fmax_legacy_ogt_f32_multi_use
 ; SI: buffer_load_dword [[A:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI: buffer_load_dword [[B:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
 ; SI-NOT: v_max_
 ; SI: v_cmp_gt_f32
 ; SI-NEXT: v_cndmask_b32
 ; SI-NOT: v_max_

test/CodeGen/AMDGPU/fmin_legacy.ll

define void @test_fmin_legacy_ult_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
   %b = load float, float addrspace(1)* %gep.1, align 4
 
   %cmp = fcmp ult float %a, %b
   %val = select i1 %cmp, float %a, float %b
   store float %val, float addrspace(1)* %out, align 4
   ret void
 }
 
+; FUNC-LABEL: {{^}}test_fmin_legacy_ult_v1f32:
+; SI: buffer_load_dword [[A:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; SI: buffer_load_dword [[B:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
+; SI-SAFE: v_min_legacy_f32_e32 {{v[0-9]+}}, [[B]], [[A]]
+; SI-NONAN: v_min_f32_e32 {{v[0-9]+}}, [[B]], [[A]]
+define void @test_fmin_legacy_ult_v1f32(<1 x float> addrspace(1)* %out, <1 x float> addrspace(1)* %in) #0 {
+  %tid = call i32 @llvm.r600.read.tidig.x() #1
+  %gep.0 = getelementptr <1 x float>, <1 x float> addrspace(1)* %in, i32 %tid
+  %gep.1 = getelementptr <1 x float>, <1 x float> addrspace(1)* %gep.0, i32 1
+
+  %a = load <1 x float>, <1 x float> addrspace(1)* %gep.0
+  %b = load <1 x float>, <1 x float> addrspace(1)* %gep.1
+
+  %cmp = fcmp ult <1 x float> %a, %b
+  %val = select <1 x i1> %cmp, <1 x float> %a, <1 x float> %b
+  store <1 x float> %val, <1 x float> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_fmin_legacy_ult_v2f32:
+; SI: buffer_load_dwordx2
+; SI: buffer_load_dwordx2
+; SI-SAFE: v_min_legacy_f32_e32
+; SI-SAFE: v_min_legacy_f32_e32
+
+; SI-NONAN: v_min_f32_e32
+; SI-NONAN: v_min_f32_e32
+define void @test_fmin_legacy_ult_v2f32(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %in) #0 {
+  %tid = call i32 @llvm.r600.read.tidig.x() #1
+  %gep.0 = getelementptr <2 x float>, <2 x float> addrspace(1)* %in, i32 %tid
+  %gep.1 = getelementptr <2 x float>, <2 x float> addrspace(1)* %gep.0, i32 1
+
+  %a = load <2 x float>, <2 x float> addrspace(1)* %gep.0
+  %b = load <2 x float>, <2 x float> addrspace(1)* %gep.1
+
+  %cmp = fcmp ult <2 x float> %a, %b
+  %val = select <2 x i1> %cmp, <2 x float> %a, <2 x float> %b
+  store <2 x float> %val, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_fmin_legacy_ult_v3f32:
+; SI-SAFE: v_min_legacy_f32_e32
+; SI-SAFE: v_min_legacy_f32_e32
+; SI-SAFE: v_min_legacy_f32_e32
+
+; SI-NONAN: v_min_f32_e32
+; SI-NONAN: v_min_f32_e32
+; SI-NONAN: v_min_f32_e32
+define void @test_fmin_legacy_ult_v3f32(<3 x float> addrspace(1)* %out, <3 x float> addrspace(1)* %in) #0 {
+  %tid = call i32 @llvm.r600.read.tidig.x() #1
+  %gep.0 = getelementptr <3 x float>, <3 x float> addrspace(1)* %in, i32 %tid
+  %gep.1 = getelementptr <3 x float>, <3 x float> addrspace(1)* %gep.0, i32 1
+
+  %a = load <3 x float>, <3 x float> addrspace(1)* %gep.0
+  %b = load <3 x float>, <3 x float> addrspace(1)* %gep.1
+
+  %cmp = fcmp ult <3 x float> %a, %b
+  %val = select <3 x i1> %cmp, <3 x float> %a, <3 x float> %b
+  store <3 x float> %val, <3 x float> addrspace(1)* %out
+  ret void
+}
+
 ; FUNC-LABEL: @test_fmin_legacy_ole_f32_multi_use
 ; SI: buffer_load_dword [[A:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI: buffer_load_dword [[B:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
 ; SI-NOT: v_min
 ; SI: v_cmp_le_f32
 ; SI-NEXT: v_cndmask_b32
 ; SI-NOT: v_min
 ; SI: s_endpgm

test/CodeGen/AMDGPU/max.ll

 ; RUN: llc -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 
 declare i32 @llvm.r600.read.tidig.x() nounwind readnone
 
-; FUNC-LABEL: @v_test_imax_sge_i32
+; FUNC-LABEL: {{^}}v_test_imax_sge_i32:
 ; SI: v_max_i32_e32
 define void @v_test_imax_sge_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
   %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
   %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
   %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
   %a = load i32, i32 addrspace(1)* %gep0, align 4
   %b = load i32, i32 addrspace(1)* %gep1, align 4
   %cmp = icmp sge i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %outgep, align 4
   ret void
 }
 
+; FUNC-LABEL: {{^}}v_test_imax_sge_v4i32:
+; SI: v_max_i32_e32
+; SI: v_max_i32_e32
+; SI: v_max_i32_e32
+; SI: v_max_i32_e32
+define void @v_test_imax_sge_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %aptr, <4 x i32> addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
+  %gep0 = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %aptr, i32 %tid
+  %gep1 = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %bptr, i32 %tid
+  %outgep = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %out, i32 %tid
+  %a = load <4 x i32>, <4 x i32> addrspace(1)* %gep0, align 4
+  %b = load <4 x i32>, <4 x i32> addrspace(1)* %gep1, align 4
+  %cmp = icmp sge <4 x i32> %a, %b
+  %val = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b
+  store <4 x i32> %val, <4 x i32> addrspace(1)* %outgep, align 4
+  ret void
+}
+
 ; FUNC-LABEL: @s_test_imax_sge_i32
 ; SI: s_max_i32
 define void @s_test_imax_sge_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
   %cmp = icmp sge i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %out, align 4
   ret void
 }

 ; SI: s_max_i32 {{s[0-9]+}}, {{s[0-9]+}}, 9
 define void @s_test_imax_sgt_imm_i32(i32 addrspace(1)* %out, i32 %a) nounwind {
   %cmp = icmp sgt i32 %a, 9
   %val = select i1 %cmp, i32 %a, i32 9
   store i32 %val, i32 addrspace(1)* %out, align 4
   ret void
 }
 
+; FUNC-LABEL: {{^}}s_test_imax_sgt_imm_v2i32:
+; SI: s_max_i32 {{s[0-9]+}}, {{s[0-9]+}}, 9
+; SI: s_max_i32 {{s[0-9]+}}, {{s[0-9]+}}, 9
+define void @s_test_imax_sgt_imm_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a) nounwind {
+  %cmp = icmp sgt <2 x i32> %a, <i32 9, i32 9>
+  %val = select <2 x i1> %cmp, <2 x i32> %a, <2 x i32> <i32 9, i32 9>
+  store <2 x i32> %val, <2 x i32> addrspace(1)* %out, align 4
+  ret void
+}
 ; FUNC-LABEL: @v_test_imax_sgt_i32
 ; SI: v_max_i32_e32
 define void @v_test_imax_sgt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
   %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
   %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
   %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
   %a = load i32, i32 addrspace(1)* %gep0, align 4

 ; SI: s_max_u32
 define void @s_test_umax_uge_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
   %cmp = icmp uge i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %out, align 4
   ret void
 }
 
+; FUNC-LABEL: {{^}}s_test_umax_uge_v3i32:
+; SI: s_max_u32
+; SI: s_max_u32
+; SI: s_max_u32
+; SI-NOT: s_max_u32
+; SI: s_endpgm
+define void @s_test_umax_uge_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> %a, <3 x i32> %b) nounwind {
+  %cmp = icmp uge <3 x i32> %a, %b
+  %val = select <3 x i1> %cmp, <3 x i32> %a, <3 x i32> %b
+  store <3 x i32> %val, <3 x i32> addrspace(1)* %out, align 4
+  ret void
+}
+
 ; FUNC-LABEL: @v_test_umax_ugt_i32
 ; SI: v_max_u32_e32
 define void @v_test_umax_ugt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
   %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
   %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
   %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
   %a = load i32, i32 addrspace(1)* %gep0, align 4
   %b = load i32, i32 addrspace(1)* %gep1, align 4
   %cmp = icmp ugt i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %outgep, align 4
   ret void
 }
 
-; FUNC-LABEL: @s_test_umax_ugt_i32
+; FUNC-LABEL: {{^}}s_test_umax_ugt_i32:
 ; SI: s_max_u32
 define void @s_test_umax_ugt_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
   %cmp = icmp ugt i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %out, align 4
   ret void
 }
 
+; FUNC-LABEL: {{^}}s_test_umax_ugt_imm_v2i32:
+; SI: s_max_u32 {{s[0-9]+}}, {{s[0-9]+}}, 15
+; SI: s_max_u32 {{s[0-9]+}}, {{s[0-9]+}}, 23
+define void @s_test_umax_ugt_imm_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a) nounwind {
+  %cmp = icmp ugt <2 x i32> %a, <i32 15, i32 23>
+  %val = select <2 x i1> %cmp, <2 x i32> %a, <2 x i32> <i32 15, i32 23>
+  store <2 x i32> %val, <2 x i32> addrspace(1)* %out, align 4
+  ret void
+}
+
 ; Make sure redundant and removed
 ; FUNC-LABEL: {{^}}simplify_demanded_bits_test_umax_ugt_i16:
 ; SI-DAG: s_load_dword [[A:s[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 0xb
 ; SI-DAG: s_load_dword [[B:s[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 0xc
-; SI: s_max_u32 [[MIN:s[0-9]+]], [[A]], [[B]]
-; SI-NEXT: v_mov_b32_e32 [[VMIN:v[0-9]+]], [[MIN]]
-; SI-NEXT: buffer_store_dword [[VMIN]]
+; SI: s_max_u32 [[MAX:s[0-9]+]], [[A]], [[B]]
+; SI-NEXT: v_mov_b32_e32 [[VMAX:v[0-9]+]], [[MAX]]
+; SI-NEXT: buffer_store_dword [[VMAX]]
 define void @simplify_demanded_bits_test_umax_ugt_i16(i32 addrspace(1)* %out, i16 zeroext %a, i16 zeroext %b) nounwind {
   %a.ext = zext i16 %a to i32
   %b.ext = zext i16 %b to i32
   %cmp = icmp ugt i32 %a.ext, %b.ext
   %val = select i1 %cmp, i32 %a.ext, i32 %b.ext
   %mask = and i32 %val, 65535
   store i32 %mask, i32 addrspace(1)* %out
   ret void
 }
 
 ; Make sure redundant sign_extend_inreg removed.
 
-; FUNC-LABEL: {{^}}simplify_demanded_bits_test_min_slt_i16:
+; FUNC-LABEL: {{^}}simplify_demanded_bits_test_max_slt_i16:
 ; SI-DAG: s_load_dword [[A:s[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 0xb
 ; SI-DAG: s_load_dword [[B:s[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 0xc
-; SI: s_max_i32 [[MIN:s[0-9]+]], [[A]], [[B]]
-; SI-NEXT: v_mov_b32_e32 [[VMIN:v[0-9]+]], [[MIN]]
-; SI-NEXT: buffer_store_dword [[VMIN]]
-define void @simplify_demanded_bits_test_min_slt_i16(i32 addrspace(1)* %out, i16 signext %a, i16 signext %b) nounwind {
+; SI: s_max_i32 [[MAX:s[0-9]+]], [[A]], [[B]]
+; SI-NEXT: v_mov_b32_e32 [[VMAX:v[0-9]+]], [[MAX]]
+; SI-NEXT: buffer_store_dword [[VMAX]]
+define void @simplify_demanded_bits_test_max_slt_i16(i32 addrspace(1)* %out, i16 signext %a, i16 signext %b) nounwind {
   %a.ext = sext i16 %a to i32
   %b.ext = sext i16 %b to i32
   %cmp = icmp sgt i32 %a.ext, %b.ext
   %val = select i1 %cmp, i32 %a.ext, i32 %b.ext
   %shl = shl i32 %val, 16
   %sextinreg = ashr i32 %shl, 16
   store i32 %sextinreg, i32 addrspace(1)* %out
   ret void
 }
 
-; FIXME: Should get match min/max through extends inserted by
-; legalization.
-
-; FUNC-LABEL: {{^}}s_test_imin_sge_i16:
-; SI: s_sext_i32_i16
-; SI: s_sext_i32_i16
-; SI: v_cmp_ge_i32_e32
-; SI: v_cndmask_b32
-define void @s_test_imin_sge_i16(i16 addrspace(1)* %out, i16 %a, i16 %b) nounwind {
+; FUNC-LABEL: {{^}}s_test_imax_sge_i16:
+; SI: s_max_i32
+define void @s_test_imax_sge_i16(i16 addrspace(1)* %out, i16 %a, i16 %b) nounwind {
   %cmp = icmp sge i16 %a, %b
   %val = select i1 %cmp, i16 %a, i16 %b
   store i16 %val, i16 addrspace(1)* %out
   ret void
 }

test/CodeGen/AMDGPU/min.ll

 ; RUN: llc -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 
 declare i32 @llvm.r600.read.tidig.x() nounwind readnone
 
-; FUNC-LABEL: @v_test_imin_sle_i32
+; FUNC-LABEL: {{^}}v_test_imin_sle_i32:
 ; SI: v_min_i32_e32
 define void @v_test_imin_sle_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
   %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
   %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
   %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
   %a = load i32, i32 addrspace(1)* %gep0, align 4
   %b = load i32, i32 addrspace(1)* %gep1, align 4
   %cmp = icmp sle i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %outgep, align 4
   ret void
 }
 
-; FUNC-LABEL: @s_test_imin_sle_i32
+; FUNC-LABEL: {{^}}s_test_imin_sle_i32:
 ; SI: s_min_i32
 define void @s_test_imin_sle_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
   %cmp = icmp sle i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %out, align 4
   ret void
 }
 
+; FUNC-LABEL: {{^}}s_test_imin_sle_v1i32:
+; SI: s_min_i32
+define void @s_test_imin_sle_v1i32(<1 x i32> addrspace(1)* %out, <1 x i32> %a, <1 x i32> %b) nounwind {
+  %cmp = icmp sle <1 x i32> %a, %b
+  %val = select <1 x i1> %cmp, <1 x i32> %a, <1 x i32> %b
+  store <1 x i32> %val, <1 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}s_test_imin_sle_v4i32:
+; SI: s_min_i32
+; SI: s_min_i32
+; SI: s_min_i32
+; SI: s_min_i32
+define void @s_test_imin_sle_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %a, <4 x i32> %b) nounwind {
+  %cmp = icmp sle <4 x i32> %a, %b
+  %val = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b
+  store <4 x i32> %val, <4 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}s_test_imin_sle_i8:
+; SI: s_min_i32
+define void @s_test_imin_sle_i8(i8 addrspace(1)* %out, i8 %a, i8 %b) nounwind {
+  %cmp = icmp sle i8 %a, %b
+  %val = select i1 %cmp, i8 %a, i8 %b
+  store i8 %val, i8 addrspace(1)* %out
+  ret void
+}
+
+; XXX - should be able to use s_min if we stop unnecessarily doing
+; extloads with mubuf instructions.
+
+; FUNC-LABEL: {{^}}s_test_imin_sle_v4i8:
+; SI: v_min_i32
+; SI: v_min_i32
+; SI: v_min_i32
+; SI: v_min_i32
+define void @s_test_imin_sle_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> %a, <4 x i8> %b) nounwind {
+  %cmp = icmp sle <4 x i8> %a, %b
+  %val = select <4 x i1> %cmp, <4 x i8> %a, <4 x i8> %b
+  store <4 x i8> %val, <4 x i8> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}s_test_imin_sle_v4i16:
+; SI: v_min_i32
+; SI: v_min_i32
+; SI: v_min_i32
+; SI: v_min_i32
+define void @s_test_imin_sle_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %a, <4 x i16> %b) nounwind {
+  %cmp = icmp sle <4 x i16> %a, %b
+  %val = select <4 x i1> %cmp, <4 x i16> %a, <4 x i16> %b
+  store <4 x i16> %val, <4 x i16> addrspace(1)* %out
+  ret void
+}
+
 ; FUNC-LABEL: @v_test_imin_slt_i32
 ; SI: v_min_i32_e32
 define void @v_test_imin_slt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
   %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
   %gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
   %gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
   %outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
   %a = load i32, i32 addrspace(1)* %gep0, align 4
   %b = load i32, i32 addrspace(1)* %gep1, align 4
   %cmp = icmp slt i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %outgep, align 4
   ret void
 }
 
 ; FUNC-LABEL: @s_test_imin_slt_i32
 ; SI: s_min_i32
 define void @s_test_imin_slt_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
   %cmp = icmp slt i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %out, align 4
   ret void
 }
 
+; FUNC-LABEL: {{^}}s_test_imin_slt_v2i32:
+; SI: s_min_i32
+; SI: s_min_i32
+define void @s_test_imin_slt_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a, <2 x i32> %b) nounwind {
+  %cmp = icmp slt <2 x i32> %a, %b
+  %val = select <2 x i1> %cmp, <2 x i32> %a, <2 x i32> %b
+  store <2 x i32> %val, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
 ; FUNC-LABEL: {{^}}s_test_imin_slt_imm_i32:
 ; SI: s_min_i32 {{s[0-9]+}}, {{s[0-9]+}}, 8
 define void @s_test_imin_slt_imm_i32(i32 addrspace(1)* %out, i32 %a) nounwind {
   %cmp = icmp slt i32 %a, 8
   %val = select i1 %cmp, i32 %a, i32 8
   store i32 %val, i32 addrspace(1)* %out, align 4
   ret void
 }
define void @v_test_umin_ule_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
   %a = load i32, i32 addrspace(1)* %gep0, align 4
   %b = load i32, i32 addrspace(1)* %gep1, align 4
   %cmp = icmp ule i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %outgep, align 4
   ret void
 }
 
+; FUNC-LABEL: @v_test_umin_ule_v3i32
+; SI: v_min_u32_e32
+; SI: v_min_u32_e32
+; SI: v_min_u32_e32
+; SI-NOT: v_min_u32_e32
+; SI: s_endpgm
+define void @v_test_umin_ule_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> addrspace(1)* %aptr, <3 x i32> addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone
+  %gep0 = getelementptr <3 x i32>, <3 x i32> addrspace(1)* %aptr, i32 %tid
+  %gep1 = getelementptr <3 x i32>, <3 x i32> addrspace(1)* %bptr, i32 %tid
+  %outgep = getelementptr <3 x i32>, <3 x i32> addrspace(1)* %out, i32 %tid
+  %a = load <3 x i32>, <3 x i32> addrspace(1)* %gep0
+  %b = load <3 x i32>, <3 x i32> addrspace(1)* %gep1
+  %cmp = icmp ule <3 x i32> %a, %b
+  %val = select <3 x i1> %cmp, <3 x i32> %a, <3 x i32> %b
+  store <3 x i32> %val, <3 x i32> addrspace(1)* %outgep
+  ret void
+}
 ; FUNC-LABEL: @s_test_umin_ule_i32
 ; SI: s_min_u32
 define void @s_test_umin_ule_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
   %cmp = icmp ule i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %out, align 4
   ret void
 }
define void @v_test_umin_ult_i32_multi_use(i32 addrspace(1)* %out0, i1 addrspace(1)* %out1, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
   %b = load i32, i32 addrspace(1)* %gep1, align 4
   %cmp = icmp ult i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %outgep0, align 4
   store i1 %cmp, i1 addrspace(1)* %outgep1
   ret void
 }
 
+
+; FUNC-LABEL: @s_test_umin_ult_v1i32
+; SI: s_min_u32
+define void @s_test_umin_ult_v1i32(<1 x i32> addrspace(1)* %out, <1 x i32> %a, <1 x i32> %b) nounwind {
+  %cmp = icmp ult <1 x i32> %a, %b
+  %val = select <1 x i1> %cmp, <1 x i32> %a, <1 x i32> %b
+  store <1 x i32> %val, <1 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}s_test_umin_ult_v8i32:
+; SI: s_min_u32
+; SI: s_min_u32
+; SI: s_min_u32
+; SI: s_min_u32
+; SI: s_min_u32
+; SI: s_min_u32
+; SI: s_min_u32
+; SI: s_min_u32
+define void @s_test_umin_ult_v8i32(<8 x i32> addrspace(1)* %out, <8 x i32> %a, <8 x i32> %b) nounwind {
+  %cmp = icmp ult <8 x i32> %a, %b
+  %val = select <8 x i1> %cmp, <8 x i32> %a, <8 x i32> %b
+  store <8 x i32> %val, <8 x i32> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}s_test_umin_ult_v8i16:
+; SI: v_min_u32
+; SI: v_min_u32
+; SI: v_min_u32
+; SI: v_min_u32
+; SI: v_min_u32
+; SI: v_min_u32
+; SI: v_min_u32
+; SI: v_min_u32
+define void @s_test_umin_ult_v8i16(<8 x i16> addrspace(1)* %out, <8 x i16> %a, <8 x i16> %b) nounwind {
+  %cmp = icmp ult <8 x i16> %a, %b
+  %val = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b
+  store <8 x i16> %val, <8 x i16> addrspace(1)* %out
+  ret void
+}
+
 ; Make sure redundant and removed
 ; FUNC-LABEL: {{^}}simplify_demanded_bits_test_umin_ult_i16:
 ; SI-DAG: s_load_dword [[A:s[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 0xb
 ; SI-DAG: s_load_dword [[B:s[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 0xc
 ; SI: s_min_u32 [[MIN:s[0-9]+]], [[A]], [[B]]
 ; SI-NEXT: v_mov_b32_e32 [[VMIN:v[0-9]+]], [[MIN]]
 ; SI-NEXT: buffer_store_dword [[VMIN]]
 define void @simplify_demanded_bits_test_umin_ult_i16(i32 addrspace(1)* %out, i16 zeroext %a, i16 zeroext %b) nounwind {
define void @simplify_demanded_bits_test_min_slt_i16(i32 addrspace(1)* %out, i16 signext %a, i16 signext %b) nounwind {
   %cmp = icmp slt i32 %a.ext, %b.ext
   %val = select i1 %cmp, i32 %a.ext, i32 %b.ext
   %shl = shl i32 %val, 16
   %sextinreg = ashr i32 %shl, 16
   store i32 %sextinreg, i32 addrspace(1)* %out
   ret void
 }
 
-; FIXME: Should get match min/max through extends inserted by
-; legalization.
-
 ; FUNC-LABEL: {{^}}s_test_imin_sle_i16:
-; SI: s_sext_i32_i16
-; SI: s_sext_i32_i16
-; SI: v_cmp_le_i32_e32
-; SI: v_cndmask_b32
+; SI: s_min_i32
 define void @s_test_imin_sle_i16(i16 addrspace(1)* %out, i16 %a, i16 %b) nounwind {
   %cmp = icmp sle i16 %a, %b
   %val = select i1 %cmp, i16 %a, i16 %b
   store i16 %val, i16 addrspace(1)* %out
   ret void
 }

test/CodeGen/AMDGPU/sminmax.ll

define void @v_abs_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %src) nounwind {
   %cond = icmp sgt i32 %val, %neg
   %res = select i1 %cond, i32 %val, i32 %neg
   %res2 = add i32 %res, 2
   store i32 %res2, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; FUNC-LABEL: {{^}}s_abs_v2i32:
-; TODO: this should use s_abs_i32
-; GCNX: s_abs_i32
-; GCNX: s_abs_i32
-; GCN: s_sub
-; GCN: s_sub
-; GCN-DAG: v_cmp_gt
-; GCN-DAG: v_cmp_gt
-; GCN-DAG: v_cndmask_b32
-; GCN-DAG: v_cndmask_b32
-; GCN: v_add_i32
-; GCN: v_add_i32
+; GCN: s_abs_i32
+; GCN: s_abs_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
 define void @s_abs_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %val) nounwind {
   %z0 = insertelement <2 x i32> undef, i32 0, i32 0
   %z1 = insertelement <2 x i32> %z0, i32 0, i32 1
   %t0 = insertelement <2 x i32> undef, i32 2, i32 0
   %t1 = insertelement <2 x i32> %t0, i32 2, i32 1
   %neg = sub <2 x i32> %z1, %val
   %cond = icmp sgt <2 x i32> %val, %neg
   %res = select <2 x i1> %cond, <2 x i32> %val, <2 x i32> %neg
   %res2 = add <2 x i32> %res, %t1
   store <2 x i32> %res2, <2 x i32> addrspace(1)* %out, align 4
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_abs_v2i32:
 ; GCN: v_sub_i32_e32 [[NEG0:v[0-9]+]], vcc, 0, [[SRC0:v[0-9]+]]
 ; GCN: v_sub_i32_e32 [[NEG1:v[0-9]+]], vcc, 0, [[SRC1:v[0-9]+]]
-; TODO: this should use v_max_i32
-; GCNX: v_max_i32_e32 {{v[0-9]+}}, [[NEG0]], [[SRC0]]
-; GCNX: v_max_i32_e32 {{v[0-9]+}}, [[NEG1]], [[SRC1]]
-; GCN-DAG: v_cmp_gt
-; GCN-DAG: v_cmp_gt
-; GCN-DAG: v_cndmask_b32
-; GCN-DAG: v_cndmask_b32
+
+; GCN: v_max_i32_e32 {{v[0-9]+}}, [[NEG0]], [[SRC0]]
+; GCN: v_max_i32_e32 {{v[0-9]+}}, [[NEG1]], [[SRC1]]
+
 ; GCN: v_add_i32
 ; GCN: v_add_i32
 define void @v_abs_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %src) nounwind {
   %z0 = insertelement <2 x i32> undef, i32 0, i32 0
   %z1 = insertelement <2 x i32> %z0, i32 0, i32 1
   %t0 = insertelement <2 x i32> undef, i32 2, i32 0
   %t1 = insertelement <2 x i32> %t0, i32 2, i32 1
   %val = load <2 x i32>, <2 x i32> addrspace(1)* %src, align 4
   %neg = sub <2 x i32> %z1, %val
   %cond = icmp sgt <2 x i32> %val, %neg
   %res = select <2 x i1> %cond, <2 x i32> %val, <2 x i32> %neg
   %res2 = add <2 x i32> %res, %t1
   store <2 x i32> %res2, <2 x i32> addrspace(1)* %out, align 4
   ret void
 }
 
 ; FUNC-LABEL: {{^}}s_abs_v4i32:
 ; TODO: this should use s_abs_i32
-; GCNX: s_abs_i32
-; GCNX: s_abs_i32
-; GCNX: s_abs_i32
-; GCNX: s_abs_i32
-; GCN: s_sub
-; GCN: s_sub
-; GCN: s_sub
-; GCN: s_sub
-; GCN-DAG: v_cmp_gt
-; GCN-DAG: v_cmp_gt
-; GCN-DAG: v_cmp_gt
-; GCN-DAG: v_cmp_gt
-; GCN-DAG: v_cndmask_b32
-; GCN-DAG: v_cndmask_b32
-; GCN-DAG: v_cndmask_b32
-; GCN-DAG: v_cndmask_b32
-; GCN: v_add_i32
-; GCN: v_add_i32
+; GCN: s_abs_i32
+; GCN: s_abs_i32
+; GCN: s_abs_i32
+; GCN: s_abs_i32
+
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
 define void @s_abs_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %val) nounwind {
   %z0 = insertelement <4 x i32> undef, i32 0, i32 0
   %z1 = insertelement <4 x i32> %z0, i32 0, i32 1
   %z2 = insertelement <4 x i32> %z1, i32 0, i32 2
   %z3 = insertelement <4 x i32> %z2, i32 0, i32 3
   %t0 = insertelement <4 x i32> undef, i32 2, i32 0
   %t1 = insertelement <4 x i32> %t0, i32 2, i32 1
   %t2 = insertelement <4 x i32> %t1, i32 2, i32 2
   %t3 = insertelement <4 x i32> %t2, i32 2, i32 3
   %neg = sub <4 x i32> %z3, %val
   %cond = icmp sgt <4 x i32> %val, %neg
   %res = select <4 x i1> %cond, <4 x i32> %val, <4 x i32> %neg
   %res2 = add <4 x i32> %res, %t3
   store <4 x i32> %res2, <4 x i32> addrspace(1)* %out, align 4
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_abs_v4i32:
 ; GCN: v_sub_i32_e32 [[NEG0:v[0-9]+]], vcc, 0, [[SRC0:v[0-9]+]]
 ; GCN: v_sub_i32_e32 [[NEG1:v[0-9]+]], vcc, 0, [[SRC1:v[0-9]+]]
 ; GCN: v_sub_i32_e32 [[NEG2:v[0-9]+]], vcc, 0, [[SRC2:v[0-9]+]]
 ; GCN: v_sub_i32_e32 [[NEG3:v[0-9]+]], vcc, 0, [[SRC3:v[0-9]+]]
-; TODO: this should use v_max_i32
-; GCNX: v_max_i32_e32 {{v[0-9]+}}, [[NEG0]], [[SRC0]]
-; GCNX: v_max_i32_e32 {{v[0-9]+}}, [[NEG1]], [[SRC1]]
-; GCNX: v_max_i32_e32 {{v[0-9]+}}, [[NEG2]], [[SRC2]]
-; GCNX: v_max_i32_e32 {{v[0-9]+}}, [[NEG3]], [[SRC3]]
-; GCN-DAG: v_cmp_gt
-; GCN-DAG: v_cmp_gt
-; GCN-DAG: v_cmp_gt
-; GCN-DAG: v_cmp_gt
-; GCN-DAG: v_cndmask_b32
-; GCN-DAG: v_cndmask_b32
-; GCN-DAG: v_cndmask_b32
-; GCN-DAG: v_cndmask_b32
+
+; GCN: v_max_i32_e32 {{v[0-9]+}}, [[NEG0]], [[SRC0]]
+; GCN: v_max_i32_e32 {{v[0-9]+}}, [[NEG1]], [[SRC1]]
+; GCN: v_max_i32_e32 {{v[0-9]+}}, [[NEG2]], [[SRC2]]
+; GCN: v_max_i32_e32 {{v[0-9]+}}, [[NEG3]], [[SRC3]]
+
 ; GCN: v_add_i32
 ; GCN: v_add_i32
 ; GCN: v_add_i32
 ; GCN: v_add_i32
 define void @v_abs_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %src) nounwind {
   %z0 = insertelement <4 x i32> undef, i32 0, i32 0
   %z1 = insertelement <4 x i32> %z0, i32 0, i32 1
   %z2 = insertelement <4 x i32> %z1, i32 0, i32 2