[SelectionDAG] Add BITCAST handling of bitwise operations to ComputeNumSignBits

[RKSimon]

Summary

We often bitcast for bitwise ops before bitcasting back to the original value type (promotion etc.):

v16i8 result = bitcast( v16i8, and( bitcast( v2i64, v16i8 x ), bitcast( v2i64, v16i8 y ) ) )

In such cases, we can safely peek through the bitcasts to determine the minimum number of sign bits.

[RKSimon]

Looking at alternative ways I might be able to do this

[RKSimon]

Abandoning, this was fixed mainly by rL316383

lib/CodeGen/SelectionDAG/SelectionDAG.cpp

case ISD::BITCAST: {
     // TODO: Handle cases other than 'sign splat' when we have a use case.
     // Requires handling of DemandedElts and Endianness.
     if ((SrcBits % VTBits) == 0) {
       assert(Op.getValueType().isVector() && "Expected bitcast to vector");
       Tmp = ComputeNumSignBits(N0, Depth + 1);
       if (Tmp == SrcBits)
         return VTBits;
     }
+
+    // We often bitcast for bitwise ops before bitcasting back to the original
+    // value type, so we can peek through the bitcasts and use the minimum
+    // number of sign bits.
+    if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
+         N0.getOpcode() == ISD::XOR) &&
+        N0.getOperand(0).getOpcode() == ISD::BITCAST &&
+        N0.getOperand(1).getOpcode() == ISD::BITCAST) {
+      SDValue N00 = N0.getOperand(0).getOperand(0);
+      SDValue N01 = N0.getOperand(1).getOperand(0);
+      if (N00.getValueType() == VT && N01.getValueType() == VT) {
+        Tmp = ComputeNumSignBits(N00, Depth + 1);
+        if (Tmp == 1)
+          return Tmp;
+        Tmp2 = ComputeNumSignBits(N01, Depth + 1);
+        FirstAnswer = std::min(Tmp, Tmp2);
+      }
+    }
+
     break;
   }
 
   case ISD::SIGN_EXTEND:
   case ISD::SIGN_EXTEND_VECTOR_INREG:
     Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
     return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
 

test/CodeGen/X86/bitcast-and-setcc-128.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE2-SSSE3,SSE2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE2-SSSE3,SSSE3
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX12,AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX12,AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512
 
 define i8 @v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c, <8 x i16> %d) {
-; SSE2-LABEL: v8i16:
-; SSE2:       # BB#0:
-; SSE2-NEXT:    pcmpgtw %xmm1, %xmm0
-; SSE2-NEXT:    pcmpgtw %xmm3, %xmm2
-; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    packuswb %xmm2, %xmm2
-; SSE2-NEXT:    pmovmskb %xmm2, %eax
-; SSE2-NEXT:    # kill: %AL<def> %AL<kill> %EAX<kill>
-; SSE2-NEXT:    ret{{[l|q]}}
-;
-; SSSE3-LABEL: v8i16:
-; SSSE3:       # BB#0:
-; SSSE3-NEXT:    pcmpgtw %xmm1, %xmm0
-; SSSE3-NEXT:    pcmpgtw %xmm3, %xmm2
-; SSSE3-NEXT:    pand %xmm0, %xmm2
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; SSSE3-NEXT:    pmovmskb %xmm2, %eax
-; SSSE3-NEXT:    # kill: %AL<def> %AL<kill> %EAX<kill>
-; SSSE3-NEXT:    ret{{[l|q]}}
+; SSE2-SSSE3-LABEL: v8i16:
+; SSE2-SSSE3:       # BB#0:
+; SSE2-SSSE3-NEXT:    pcmpgtw %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    pcmpgtw %xmm3, %xmm2
+; SSE2-SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSE2-SSSE3-NEXT:    packsswb %xmm0, %xmm2
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm2, %eax
+; SSE2-SSSE3-NEXT:    # kill: %AL<def> %AL<kill> %EAX<kill>
+; SSE2-SSSE3-NEXT:    ret{{[l|q]}}
 ;
 ; AVX12-LABEL: v8i16:
 ; AVX12:       # BB#0:
 ; AVX12-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
 ; AVX12-NEXT:    vpcmpgtw %xmm3, %xmm2, %xmm1
 ; AVX12-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX12-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX12-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX12-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX12-NEXT:    # kill: %AL<def> %AL<kill> %EAX<kill>
 ; AVX12-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512-LABEL: v8i16:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vpcmpgtw %xmm1, %xmm0, %k1
 ; AVX512-NEXT:    vpcmpgtw %xmm3, %xmm2, %k0 {%k1}
; AVX512-NEXT:    ret{{[l|q]}}
   %x0 = icmp sgt <4 x i16> %a, %b
   %x1 = icmp sgt <4 x i16> %c, %d
   %y = and <4 x i1> %x0, %x1
   %res = bitcast <4 x i1> %y to i4
   ret i4 %res
 }
 
 define i8 @v8i8(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
-; SSE2-LABEL: v8i8:
-; SSE2:       # BB#0:
-; SSE2-NEXT:    psllw $8, %xmm3
-; SSE2-NEXT:    psraw $8, %xmm3
-; SSE2-NEXT:    psllw $8, %xmm2
-; SSE2-NEXT:    psraw $8, %xmm2
-; SSE2-NEXT:    pcmpgtw %xmm3, %xmm2
-; SSE2-NEXT:    psllw $8, %xmm1
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    psllw $8, %xmm0
-; SSE2-NEXT:    psraw $8, %xmm0
-; SSE2-NEXT:    pcmpgtw %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    # kill: %AL<def> %AL<kill> %EAX<kill>
-; SSE2-NEXT:    ret{{[l|q]}}
-;
-; SSSE3-LABEL: v8i8:
-; SSSE3:       # BB#0:
-; SSSE3-NEXT:    psllw $8, %xmm3
-; SSSE3-NEXT:    psraw $8, %xmm3
-; SSSE3-NEXT:    psllw $8, %xmm2
-; SSSE3-NEXT:    psraw $8, %xmm2
-; SSSE3-NEXT:    pcmpgtw %xmm3, %xmm2
-; SSSE3-NEXT:    psllw $8, %xmm1
-; SSSE3-NEXT:    psraw $8, %xmm1
-; SSSE3-NEXT:    psllw $8, %xmm0
-; SSSE3-NEXT:    psraw $8, %xmm0
-; SSSE3-NEXT:    pcmpgtw %xmm1, %xmm0
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; SSSE3-NEXT:    pmovmskb %xmm0, %eax
-; SSSE3-NEXT:    # kill: %AL<def> %AL<kill> %EAX<kill>
-; SSSE3-NEXT:    ret{{[l|q]}}
+; SSE2-SSSE3-LABEL: v8i8:
+; SSE2-SSSE3:       # BB#0:
+; SSE2-SSSE3-NEXT:    psllw $8, %xmm3
+; SSE2-SSSE3-NEXT:    psraw $8, %xmm3
+; SSE2-SSSE3-NEXT:    psllw $8, %xmm2
+; SSE2-SSSE3-NEXT:    psraw $8, %xmm2
+; SSE2-SSSE3-NEXT:    pcmpgtw %xmm3, %xmm2
+; SSE2-SSSE3-NEXT:    psllw $8, %xmm1
+; SSE2-SSSE3-NEXT:    psraw $8, %xmm1
+; SSE2-SSSE3-NEXT:    psllw $8, %xmm0
+; SSE2-SSSE3-NEXT:    psraw $8, %xmm0
+; SSE2-SSSE3-NEXT:    pcmpgtw %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSE2-SSSE3-NEXT:    packsswb %xmm0, %xmm0
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm0, %eax
+; SSE2-SSSE3-NEXT:    # kill: %AL<def> %AL<kill> %EAX<kill>
+; SSE2-SSSE3-NEXT:    ret{{[l|q]}}
 ;
 ; AVX12-LABEL: v8i8:
 ; AVX12:       # BB#0:
 ; AVX12-NEXT:    vpsllw $8, %xmm3, %xmm3
 ; AVX12-NEXT:    vpsraw $8, %xmm3, %xmm3
 ; AVX12-NEXT:    vpsllw $8, %xmm2, %xmm2
 ; AVX12-NEXT:    vpsraw $8, %xmm2, %xmm2
 ; AVX12-NEXT:    vpcmpgtw %xmm3, %xmm2, %xmm2
 ; AVX12-NEXT:    vpsllw $8, %xmm1, %xmm1
 ; AVX12-NEXT:    vpsraw $8, %xmm1, %xmm1
 ; AVX12-NEXT:    vpsllw $8, %xmm0, %xmm0
 ; AVX12-NEXT:    vpsraw $8, %xmm0, %xmm0
 ; AVX12-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
 ; AVX12-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX12-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX12-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX12-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX12-NEXT:    # kill: %AL<def> %AL<kill> %EAX<kill>
 ; AVX12-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512-LABEL: v8i8:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vpsllw $8, %xmm3, %xmm3
 ; AVX512-NEXT:    vpsraw $8, %xmm3, %xmm3

test/CodeGen/X86/bitcast-and-setcc-256.ll

; AVX512-NEXT:    ret{{[l|q]}}
   %x0 = icmp sgt <16 x i16> %a, %b
   %x1 = icmp sgt <16 x i16> %c, %d
   %y = and <16 x i1> %x0, %x1
   %res = bitcast <16 x i1> %y to i16
   ret i16 %res
 }
 
 define i8 @v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) {
-; SSE2-LABEL: v8i32:
-; SSE2:       # BB#0:
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE2-NEXT:    packssdw %xmm5, %xmm4
-; SSE2-NEXT:    pand %xmm0, %xmm4
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm4
-; SSE2-NEXT:    packuswb %xmm4, %xmm4
-; SSE2-NEXT:    pmovmskb %xmm4, %eax
-; SSE2-NEXT:    # kill: %AL<def> %AL<kill> %EAX<kill>
-; SSE2-NEXT:    ret{{[l|q]}}
-;
-; SSSE3-LABEL: v8i32:
-; SSSE3:       # BB#0:
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSSE3-NEXT:    packssdw %xmm1, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSSE3-NEXT:    packssdw %xmm5, %xmm4
-; SSSE3-NEXT:    pand %xmm0, %xmm4
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm4 = xmm4[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; SSSE3-NEXT:    pmovmskb %xmm4, %eax
-; SSSE3-NEXT:    # kill: %AL<def> %AL<kill> %EAX<kill>
-; SSSE3-NEXT:    ret{{[l|q]}}
+; SSE2-SSSE3-LABEL: v8i32:
+; SSE2-SSSE3:       # BB#0:
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE2-SSSE3-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm7, %xmm5
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE2-SSSE3-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-SSSE3-NEXT:    pand %xmm0, %xmm4
+; SSE2-SSSE3-NEXT:    packsswb %xmm0, %xmm4
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm4, %eax
+; SSE2-SSSE3-NEXT:    # kill: %AL<def> %AL<kill> %EAX<kill>
+; SSE2-SSSE3-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1-LABEL: v8i32:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
 ; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm4, %xmm0, %xmm0

test/CodeGen/X86/widen_arith-2.ll

 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-NEXT:    addl %ecx, %edx
 ; CHECK-NEXT:    movl %edx, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    addl {{[0-9]+}}(%esp), %ecx
 ; CHECK-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    pmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
 ; CHECK-NEXT:    psubw %xmm0, %xmm2
 ; CHECK-NEXT:    pand %xmm1, %xmm2
-; CHECK-NEXT:    packsswb %xmm0, %xmm2
+; CHECK-NEXT:    packuswb %xmm0, %xmm2
 ; CHECK-NEXT:    movq %xmm2, (%edx,%eax,8)
 ; CHECK-NEXT:    incl (%esp)
 ; CHECK-NEXT:  .LBB0_1: # %forcond
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl (%esp), %eax
 ; CHECK-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    jl .LBB0_2
 ; CHECK-NEXT:  # BB#3: # %afterfor