Index: llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -3297,7 +3297,10 @@ // by skipping the simplification if isIntDivCheap(). When div is not cheap, // combine will not return a DIVREM. Regardless, checking cheapness here // makes sense since the simplification results in fatter code. - if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap(VT, Attr)) { + // TODO: replace matchUnaryPredicate with SelectionDAG::isKnownNeverZero(N1). + if (ISD::matchUnaryPredicate( + N1, [](ConstantSDNode *C) { return !C->isNullValue(); }) && + !TLI.isIntDivCheap(VT, Attr)) { SDValue OptimizedDiv = isSigned ? visitSDIVLike(N0, N1, N) : visitUDIVLike(N0, N1, N); if (OptimizedDiv.getNode() && OptimizedDiv.getOpcode() != ISD::UDIVREM && Index: llvm/trunk/test/CodeGen/X86/combine-srem.ll =================================================================== --- llvm/trunk/test/CodeGen/X86/combine-srem.ll +++ llvm/trunk/test/CodeGen/X86/combine-srem.ll @@ -274,61 +274,64 @@ define <4 x i32> @combine_vec_srem_by_pow2b(<4 x i32> %x) { ; SSE-LABEL: combine_vec_srem_by_pow2b: ; SSE: # %bb.0: -; SSE-NEXT: extractps $3, %xmm0, %eax -; SSE-NEXT: movl %eax, %ecx -; SSE-NEXT: sarl $31, %ecx -; SSE-NEXT: shrl $29, %ecx -; SSE-NEXT: addl %eax, %ecx -; SSE-NEXT: andl $-8, %ecx -; SSE-NEXT: subl %ecx, %eax -; SSE-NEXT: movd %eax, %xmm1 -; SSE-NEXT: extractps $2, %xmm0, %eax -; SSE-NEXT: movl %eax, %ecx -; SSE-NEXT: sarl $31, %ecx -; SSE-NEXT: shrl $30, %ecx -; SSE-NEXT: addl %eax, %ecx -; SSE-NEXT: andl $-4, %ecx -; SSE-NEXT: subl %ecx, %eax -; SSE-NEXT: movd %eax, %xmm2 -; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSE-NEXT: extractps $1, %xmm0, %eax -; SSE-NEXT: movl %eax, %ecx -; SSE-NEXT: shrl $31, %ecx -; SSE-NEXT: addl %eax, %ecx -; SSE-NEXT: andl $-2, %ecx -; SSE-NEXT: subl %ecx, %eax -; SSE-NEXT: movd %eax, %xmm0 -; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm2[0,1] +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psrld $31, %xmm1 +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: psrad $31, %xmm2 +; SSE-NEXT: movdqa %xmm2, %xmm3 +; SSE-NEXT: psrld $29, %xmm3 +; SSE-NEXT: pblendw {{.*#+}} xmm3 = xmm1[0,1,2,3],xmm3[4,5,6,7] +; SSE-NEXT: psrld $30, %xmm2 +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7] +; SSE-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7] +; SSE-NEXT: paddd %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: movdqa %xmm1, %xmm3 +; SSE-NEXT: psrad $2, %xmm3 +; SSE-NEXT: pblendw {{.*#+}} xmm3 = xmm1[0,1,2,3],xmm3[4,5,6,7] +; SSE-NEXT: psrad $3, %xmm1 +; SSE-NEXT: psrad $1, %xmm2 +; SSE-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7] +; SSE-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7] +; SSE-NEXT: pblendw {{.*#+}} xmm3 = xmm0[0,1],xmm3[2,3,4,5,6,7] +; SSE-NEXT: pmulld {{.*}}(%rip), %xmm3 +; SSE-NEXT: psubd %xmm3, %xmm0 ; SSE-NEXT: retq ; -; AVX-LABEL: combine_vec_srem_by_pow2b: -; AVX: # %bb.0: -; AVX-NEXT: vextractps $3, %xmm0, %eax -; AVX-NEXT: movl %eax, %ecx -; AVX-NEXT: sarl $31, %ecx -; AVX-NEXT: shrl $29, %ecx -; AVX-NEXT: addl %eax, %ecx -; AVX-NEXT: andl $-8, %ecx -; AVX-NEXT: subl %ecx, %eax -; AVX-NEXT: vmovd %eax, %xmm1 -; AVX-NEXT: vextractps $2, %xmm0, %eax -; AVX-NEXT: movl %eax, %ecx -; AVX-NEXT: sarl $31, %ecx -; AVX-NEXT: shrl $30, %ecx -; AVX-NEXT: addl %eax, %ecx -; AVX-NEXT: andl $-4, %ecx -; AVX-NEXT: subl %ecx, %eax -; AVX-NEXT: vmovd %eax, %xmm2 -; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; AVX-NEXT: vextractps $1, %xmm0, %eax -; AVX-NEXT: movl %eax, %ecx -; AVX-NEXT: shrl $31, %ecx -; AVX-NEXT: addl %eax, %ecx -; AVX-NEXT: andl $-2, %ecx -; AVX-NEXT: subl %ecx, %eax -; AVX-NEXT: vmovd %eax, %xmm0 -; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,0],xmm1[0,1] -; AVX-NEXT: retq +; AVX1-LABEL: combine_vec_srem_by_pow2b: +; AVX1: # %bb.0: +; AVX1-NEXT: vpsrld $31, %xmm0, %xmm1 +; AVX1-NEXT: vpsrad $31, %xmm0, %xmm2 +; AVX1-NEXT: vpsrld $29, %xmm2, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5,6,7] +; AVX1-NEXT: vpsrld $30, %xmm2, %xmm2 +; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7] +; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm1 +; AVX1-NEXT: vpsrad $3, %xmm1, %xmm2 +; AVX1-NEXT: vpsrad $1, %xmm1, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vpsrad $2, %xmm1, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] +; AVX1-NEXT: vpmulld {{.*}}(%rip), %xmm1, %xmm1 +; AVX1-NEXT: vpsubd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: combine_vec_srem_by_pow2b: +; AVX2: # %bb.0: +; AVX2-NEXT: vpsrad $31, %xmm0, %xmm1 +; AVX2-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 +; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm1 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3] +; AVX2-NEXT: vpsravd %xmm2, %xmm1, %xmm1 +; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3] +; AVX2-NEXT: vpsllvd %xmm2, %xmm1, %xmm1 +; AVX2-NEXT: vpsubd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq %1 = srem <4 x i32> %x, ret <4 x i32> %1 } @@ -336,98 +339,70 @@ define <4 x i32> @combine_vec_srem_by_pow2b_neg(<4 x i32> %x) { ; SSE-LABEL: combine_vec_srem_by_pow2b_neg: ; SSE: # %bb.0: -; SSE-NEXT: pextrd $1, %xmm0, %eax -; SSE-NEXT: movl %eax, %ecx -; SSE-NEXT: sarl $31, %ecx -; SSE-NEXT: shrl $30, %ecx -; SSE-NEXT: addl %eax, %ecx -; SSE-NEXT: shrl $2, %ecx -; SSE-NEXT: negl %ecx -; SSE-NEXT: shll $2, %ecx -; SSE-NEXT: negl %ecx -; SSE-NEXT: subl %ecx, %eax -; SSE-NEXT: movd %xmm0, %ecx -; SSE-NEXT: movl %ecx, %edx -; SSE-NEXT: shrl $31, %edx -; SSE-NEXT: addl %ecx, %edx -; SSE-NEXT: shrl %edx -; SSE-NEXT: negl %edx -; SSE-NEXT: addl %edx, %edx -; SSE-NEXT: negl %edx -; SSE-NEXT: subl %edx, %ecx -; SSE-NEXT: movd %ecx, %xmm1 -; SSE-NEXT: pinsrd $1, %eax, %xmm1 -; SSE-NEXT: pextrd $2, %xmm0, %eax -; SSE-NEXT: movl %eax, %ecx -; SSE-NEXT: sarl $31, %ecx -; SSE-NEXT: shrl $29, %ecx -; SSE-NEXT: addl %eax, %ecx -; SSE-NEXT: shrl $3, %ecx -; SSE-NEXT: negl %ecx -; SSE-NEXT: shll $3, %ecx -; SSE-NEXT: negl %ecx -; SSE-NEXT: subl %ecx, %eax -; SSE-NEXT: pinsrd $2, %eax, %xmm1 -; SSE-NEXT: pextrd $3, %xmm0, %eax -; SSE-NEXT: movl %eax, %ecx -; SSE-NEXT: sarl $31, %ecx -; SSE-NEXT: shrl $28, %ecx -; SSE-NEXT: addl %eax, %ecx -; SSE-NEXT: shrl $4, %ecx -; SSE-NEXT: negl %ecx -; SSE-NEXT: shll $4, %ecx -; SSE-NEXT: negl %ecx -; SSE-NEXT: subl %ecx, %eax -; SSE-NEXT: pinsrd $3, %eax, %xmm1 -; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psrad $31, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: psrld $28, %xmm2 +; SSE-NEXT: movdqa %xmm1, %xmm3 +; SSE-NEXT: psrld $30, %xmm3 +; SSE-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7] +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: psrld $31, %xmm2 +; SSE-NEXT: psrld $29, %xmm1 +; SSE-NEXT: pblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7] +; SSE-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7] +; SSE-NEXT: paddd %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: psrad $4, %xmm2 +; SSE-NEXT: movdqa %xmm1, %xmm3 +; SSE-NEXT: psrad $2, %xmm3 +; SSE-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7] +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: psrad $3, %xmm2 +; SSE-NEXT: psrad $1, %xmm1 +; SSE-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7] +; SSE-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7] +; SSE-NEXT: pxor %xmm2, %xmm2 +; SSE-NEXT: psubd %xmm1, %xmm2 +; SSE-NEXT: pmulld {{.*}}(%rip), %xmm2 +; SSE-NEXT: psubd %xmm2, %xmm0 ; SSE-NEXT: retq ; -; AVX-LABEL: combine_vec_srem_by_pow2b_neg: -; AVX: # %bb.0: -; AVX-NEXT: vpextrd $1, %xmm0, %eax -; AVX-NEXT: movl %eax, %ecx -; AVX-NEXT: sarl $31, %ecx -; AVX-NEXT: shrl $30, %ecx -; AVX-NEXT: addl %eax, %ecx -; AVX-NEXT: shrl $2, %ecx -; AVX-NEXT: negl %ecx -; AVX-NEXT: shll $2, %ecx -; AVX-NEXT: negl %ecx -; AVX-NEXT: subl %ecx, %eax -; AVX-NEXT: vmovd %xmm0, %ecx -; AVX-NEXT: movl %ecx, %edx -; AVX-NEXT: shrl $31, %edx -; AVX-NEXT: addl %ecx, %edx -; AVX-NEXT: shrl %edx -; AVX-NEXT: negl %edx -; AVX-NEXT: addl %edx, %edx -; AVX-NEXT: negl %edx -; AVX-NEXT: subl %edx, %ecx -; AVX-NEXT: vmovd %ecx, %xmm1 -; AVX-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrd $2, %xmm0, %eax -; AVX-NEXT: movl %eax, %ecx -; AVX-NEXT: sarl $31, %ecx -; AVX-NEXT: shrl $29, %ecx -; AVX-NEXT: addl %eax, %ecx -; AVX-NEXT: shrl $3, %ecx -; AVX-NEXT: negl %ecx -; AVX-NEXT: shll $3, %ecx -; AVX-NEXT: negl %ecx -; AVX-NEXT: subl %ecx, %eax -; AVX-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrd $3, %xmm0, %eax -; AVX-NEXT: movl %eax, %ecx -; AVX-NEXT: sarl $31, %ecx -; AVX-NEXT: shrl $28, %ecx -; AVX-NEXT: addl %eax, %ecx -; AVX-NEXT: shrl $4, %ecx -; AVX-NEXT: negl %ecx -; AVX-NEXT: shll $4, %ecx -; AVX-NEXT: negl %ecx -; AVX-NEXT: subl %ecx, %eax -; AVX-NEXT: vpinsrd $3, %eax, %xmm1, %xmm0 -; AVX-NEXT: retq +; AVX1-LABEL: combine_vec_srem_by_pow2b_neg: +; AVX1: # %bb.0: +; AVX1-NEXT: vpsrad $31, %xmm0, %xmm1 +; AVX1-NEXT: vpsrld $28, %xmm1, %xmm2 +; AVX1-NEXT: vpsrld $30, %xmm1, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vpsrld $31, %xmm0, %xmm3 +; AVX1-NEXT: vpsrld $29, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7] +; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm1 +; AVX1-NEXT: vpsrad $4, %xmm1, %xmm2 +; AVX1-NEXT: vpsrad $2, %xmm1, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vpsrad $3, %xmm1, %xmm3 +; AVX1-NEXT: vpsrad $1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7] +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpsubd %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpmulld {{.*}}(%rip), %xmm1, %xmm1 +; AVX1-NEXT: vpsubd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: combine_vec_srem_by_pow2b_neg: +; AVX2: # %bb.0: +; AVX2-NEXT: vpsrad $31, %xmm0, %xmm1 +; AVX2-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 +; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm1 +; AVX2-NEXT: vpsravd {{.*}}(%rip), %xmm1, %xmm1 +; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX2-NEXT: vpsubd %xmm1, %xmm2, %xmm1 +; AVX2-NEXT: vpmulld {{.*}}(%rip), %xmm1, %xmm1 +; AVX2-NEXT: vpsubd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq %1 = srem <4 x i32> %x, ret <4 x i32> %1 }