AVX-512: Fixed a crash during legalization of <3 x i8> type

[delena]

Summary

The compiler fails with assertion during legalization of SETCC for <3 x i8> operands.
The result is extended to <4 x i8> and then truncated <4 x i1>. It does not happen on AVX2, because the final result of SETCC is <4 x i32>.

[RKSimon]

lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

622

Merge into DAG.getSExtOrTrunc ?

test/CodeGen/X86/avx512-vec3-crash.ll

2

I don't see any reason why you shouldn't FileCheck this and generate the codegen with update_llc_test_checks.py

[delena]

Code changes according to Simon's comments.

[RKSimon]

LGTM, thanks

[delena]

Thanks, Simon!

lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger &&
         !RHS.getValueType().isVector())
       RHS = GetPromotedInteger(RHS);
   }
 
   // Get the SETCC result using the canonical SETCC type.
   SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, LHS, RHS,
                               N->getOperand(2));
 
-  assert(NVT.bitsLE(SVT) && "Integer type overpromoted?");
   // Convert to the expected type.
-  return DAG.getNode(ISD::TRUNCATE, dl, NVT, SetCC);
+  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
   if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
     LHS = GetPromotedInteger(LHS);
   if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
     RHS = ZExtPromotedInteger(RHS);
   return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
 }

test/CodeGen/X86/avx512-vec3-crash.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=skx -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+; This test crashed during type legalization of SETCC result type.
+define <3 x i8 > @foo(<3 x i8>%x, <3 x i8>%a, <3 x i8>%b) {
+; CHECK-LABEL: foo:
+; CHECK:       # BB#0:
+; CHECK-NEXT:    vmovd %edi, %xmm0
+; CHECK-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
+; CHECK-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
+; CHECK-NEXT:    vpslld $24, %xmm0, %xmm0
+; CHECK-NEXT:    vpsrad $24, %xmm0, %xmm0
+; CHECK-NEXT:    vmovd %ecx, %xmm1
+; CHECK-NEXT:    vpinsrd $1, %r8d, %xmm1, %xmm1
+; CHECK-NEXT:    vpinsrd $2, %r9d, %xmm1, %xmm1
+; CHECK-NEXT:    vpslld $24, %xmm1, %xmm1
+; CHECK-NEXT:    vpsrad $24, %xmm1, %xmm1
+; CHECK-NEXT:    vpcmpgtd %xmm0, %xmm1, %k0
+; CHECK-NEXT:    vpmovm2d %k0, %xmm0
+; CHECK-NEXT:    vpextrb $0, %xmm0, %eax
+; CHECK-NEXT:    vpextrb $4, %xmm0, %edx
+; CHECK-NEXT:    vpextrb $8, %xmm0, %ecx
+; CHECK-NEXT:    # kill: %AL<def> %AL<kill> %EAX<kill>
+; CHECK-NEXT:    # kill: %DL<def> %DL<kill> %EDX<kill>
+; CHECK-NEXT:    # kill: %CL<def> %CL<kill> %ECX<kill>
+; CHECK-NEXT:    retq
+  %cmp.i = icmp slt <3 x i8> %x, %a
+  %res = sext <3 x i1> %cmp.i to <3 x i8>
+  ret <3 x i8> %res
+}
+