[X86] Add tests for checking `isKnownNeverZero`; NFC

[RKSimon]

llvm/test/CodeGen/X86/known-never-zero.ll

57

Any idea why we sometimes get branches and sometimes CMOV? IIRC CodeGenPrepare does attempt to determine if a CTTZ/CTLZ is non-zero?

289

use the fshl/fshr intrinsics here to help ensure you're testing the ROTL/ROTR paths

[goldstein.w.n]

llvm/test/CodeGen/X86/known-never-zero.ll

57

I'm not sure. The transform happens in: visitCTTZ. See my comment on fshr, but I think there is some inter-dependency with the middle end that might be subtly affecting things.

68

As an aside, this is a todo:

Cmp(Cmov(CC, Zero, Non-Zero), Zero) should be (Cmp(CC))

289

I originally did this, but as I was working on the change I noticed we are also missing non-zero case for funnel-shift in the middle end. I filled in the middle-end case and found it was affecting the CodeGen test and figured it be best to avoid that kind of inter-dependency. Changes to ValueTracking.cpp shouldn't mess with a test in CodeGen/X86.

Maybe different llc flags?

[RKSimon]

llvm/test/CodeGen/X86/known-never-zero.ll

289

CodeGenPrepare (despeculateCountZeros) is trying to prevent CTLZ/CTTZ zero-input branching from appearing, using ValueTracking's variant of isKnownNonZero

I wonder if there's something other than CTTZ we could use?

[goldstein.w.n]

llvm/test/CodeGen/X86/known-never-zero.ll

289

I see. We don't use it in many places. Let me post a patch to try and evaluate SetCC based on known bits, then we can use that.

[goldstein.w.n]

llvm/test/CodeGen/X86/known-never-zero.ll

289

CodeGenPrepare (despeculateCountZeros) is trying to prevent CTLZ/CTTZ zero-input branching from appearing, using ValueTracking's variant of isKnownNonZero

I wonder if there's something other than CTTZ we could use?

Created: D149383, once that goes through I'll update tests to use compare against zero.

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

[goldstein.w.n]

rebase

[goldstein.w.n]

Rebase

[RKSimon]

LGTM (although I think you still need rotl/rotr coverage?)

[goldstein.w.n]

In D149299#4484567, @RKSimon wrote:

LGTM (although I think you still need rotl/rotr coverage?)

Added fshl and fshr cases as well. But we do have:

define i32 @rotr_known_nonzero(i32 %xx, i32 %y) {
; CHECK-LABEL: rotr_known_nonzero:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movl %esi, %ecx
; CHECK-NEXT:    orl $256, %edi # imm = 0x100
; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
; CHECK-NEXT:    rorl %cl, %edi
; CHECK-NEXT:    testl %edi, %edi
; CHECK-NEXT:    je .LBB14_1
; CHECK-NEXT:  # %bb.2: # %cond.false
; CHECK-NEXT:    rep bsfl %edi, %eax
; CHECK-NEXT:    retq
; CHECK-NEXT:  .LBB14_1:
; CHECK-NEXT:    movl $32, %eax
; CHECK-NEXT:    retq
  %x = or i32 %xx, 256
  %shr = lshr i32 %x, %y
  %sub = sub i32 32, %y
  %shl = shl i32 %x, %sub
  %z = or i32 %shl, %shr
  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
  ret i32 %r
}

define i32 @rotr_maybe_zero(i32 %x, i32 %y) {
; CHECK-LABEL: rotr_maybe_zero:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movl %esi, %ecx
; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
; CHECK-NEXT:    rorl %cl, %edi
; CHECK-NEXT:    testl %edi, %edi
; CHECK-NEXT:    je .LBB15_1
; CHECK-NEXT:  # %bb.2: # %cond.false
; CHECK-NEXT:    rep bsfl %edi, %eax
; CHECK-NEXT:    retq
; CHECK-NEXT:  .LBB15_1:
; CHECK-NEXT:    movl $32, %eax
; CHECK-NEXT:    retq
  %shr = lshr i32 %x, %y
  %sub = sub i32 32, %y
  %shl = shl i32 %x, %sub
  %z = or i32 %shl, %shr
  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
  ret i32 %r
}

Which are generating the rotate instructions at the very least.

[goldstein.w.n]

Rebase

[RKSimon]

LGTM - cheers

llvm/test/CodeGen/X86/known-never-zero.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=CHECK
+
+;; Use cttz to test if we properly prove never-zero. There is a very
+;; simple transform from cttz -> cttz_zero_undef if its operand is
+;; known never zero.
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i32 @llvm.uadd.sat.i32(i32, i32)
+declare i32 @llvm.umax.i32(i32, i32)
+declare i32 @llvm.umin.i32(i32, i32)
+declare i32 @llvm.bswap.i32(i32)
+declare i32 @llvm.bitreverse.i32(i32)
+declare i32 @llvm.ctpop.i32(i32)
+declare i32 @llvm.abs.i32(i32, i1)
+declare i32 @llvm.fshl.i32(i32, i32, i32)
+declare i32 @llvm.fshr.i32(i32, i32, i32)
+
+define i32 @or_known_nonzero(i32 %x) {
+; CHECK-LABEL: or_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    orl $1, %edi
+; CHECK-NEXT:    rep bsfl %edi, %eax
+; CHECK-NEXT:    retq
+  %z = or i32 %x, 1
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @or_maybe_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: or_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    orl %esi, %edi
+; CHECK-NEXT:    je .LBB1_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %edi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB1_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = or i32 %x, %y
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @select_known_nonzero(i1 %c, i32 %x) {
+; CHECK-LABEL: select_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    orl $1, %esi
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    movl $122, %eax
+; CHECK-NEXT:    cmovnel %esi, %eax
+; CHECK-NEXT:    bsfl %eax, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %y = or i32 %x, 1
+  %z = select i1 %c, i32 %y, i32 122
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @select_maybe_zero(i1 %c, i32 %x) {
+; CHECK-LABEL: select_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    orl $1, %esi
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    cmovnel %esi, %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    je .LBB3_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %eax, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB3_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %y = or i32 %x, 1
+  %z = select i1 %c, i32 %y, i32 0
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @shl_known_nonzero_1s_bit_set(i32 %x) {
+; CHECK-LABEL: shl_known_nonzero_1s_bit_set:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    movl $123, %eax
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shll %cl, %eax
+; CHECK-NEXT:    bsfl %eax, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %z = shl i32 123, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @shl_known_nonzero_nsw(i32 %x, i32 %yy) {
+; CHECK-LABEL: shl_known_nonzero_nsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    orl $256, %esi # imm = 0x100
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shll %cl, %esi
+; CHECK-NEXT:    bsfl %esi, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %y = or i32 %yy, 256
+  %z = shl nsw i32 %y, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @shl_known_nonzero_nuw(i32 %x, i32 %yy) {
+; CHECK-LABEL: shl_known_nonzero_nuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    orl $256, %esi # imm = 0x100
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shll %cl, %esi
+; CHECK-NEXT:    bsfl %esi, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %y = or i32 %yy, 256
+  %z = shl nuw i32 %y, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @shl_maybe_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: shl_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shll %cl, %esi
+; CHECK-NEXT:    testl %esi, %esi
+; CHECK-NEXT:    je .LBB7_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %esi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB7_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = shl nuw nsw i32 %y, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @uaddsat_known_nonzero(i32 %x) {
+; CHECK-LABEL: uaddsat_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    incl %edi
+; CHECK-NEXT:    movl $-1, %eax
+; CHECK-NEXT:    cmovnel %edi, %eax
+; CHECK-NEXT:    bsfl %eax, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %z = call i32 @llvm.uadd.sat.i32(i32 %x, i32 1)
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @uaddsat_maybe_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: uaddsat_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addl %esi, %edi
+; CHECK-NEXT:    movl $-1, %eax
+; CHECK-NEXT:    cmovael %edi, %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    je .LBB9_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %eax, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB9_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y)
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @umax_known_nonzero(i32 %x, i32 %y) {
+; CHECK-LABEL: umax_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    movl $4, %eax
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shll %cl, %eax
+; CHECK-NEXT:    cmpl %eax, %edi
+; CHECK-NEXT:    cmoval %edi, %eax
+; CHECK-NEXT:    bsfl %eax, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %yy = shl nuw i32 4, %y
+  %z = call i32 @llvm.umax.i32(i32 %x, i32 %yy)
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @umax_maybe_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: umax_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cmpl %esi, %edi
+; CHECK-NEXT:    cmoval %edi, %esi
+; CHECK-NEXT:    testl %esi, %esi
+; CHECK-NEXT:    je .LBB11_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %esi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB11_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = call i32 @llvm.umax.i32(i32 %x, i32 %y)
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @umin_known_nonzero(i32 %xx, i32 %yy) {
+; CHECK-LABEL: umin_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    movl $4, %eax
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shll %cl, %eax
+; CHECK-NEXT:    addl $4, %esi
+; CHECK-NEXT:    cmpl %esi, %eax
+; CHECK-NEXT:    cmovbl %eax, %esi
+; CHECK-NEXT:    bsfl %esi, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %x = shl nuw i32 4, %xx
+  %y = add nuw nsw i32 %yy, 4
+  %z = call i32 @llvm.umin.i32(i32 %x, i32 %y)
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @umin_maybe_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: umin_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cmpl $54, %edi
+; CHECK-NEXT:    movl $54, %eax
+; CHECK-NEXT:    cmovbl %edi, %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    je .LBB13_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %eax, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB13_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = call i32 @llvm.umin.i32(i32 %x, i32 54)
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @rotr_known_nonzero(i32 %xx, i32 %y) {
+; CHECK-LABEL: rotr_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    orl $256, %edi # imm = 0x100
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    rorl %cl, %edi
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB14_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %edi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB14_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %x = or i32 %xx, 256
+  %shr = lshr i32 %x, %y
+  %sub = sub i32 32, %y
+  %shl = shl i32 %x, %sub
+  %z = or i32 %shl, %shr
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @rotr_maybe_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: rotr_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    rorl %cl, %edi
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB15_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %edi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB15_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %shr = lshr i32 %x, %y
+  %sub = sub i32 32, %y
+  %shl = shl i32 %x, %sub
+  %z = or i32 %shl, %shr
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @rotr_with_fshr_known_nonzero(i32 %xx, i32 %y) {
+; CHECK-LABEL: rotr_with_fshr_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    orl $256, %edi # imm = 0x100
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    rorl %cl, %edi
+; CHECK-NEXT:    bsfl %edi, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %x = or i32 %xx, 256
+  %z = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 %y)
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @rotr_with_fshr_maybe_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: rotr_with_fshr_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    rorl %cl, %edi
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB17_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %edi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB17_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 %y)
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @rotl_known_nonzero(i32 %xx, i32 %y) {
+; CHECK-LABEL: rotl_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    orl $256, %edi # imm = 0x100
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    roll %cl, %edi
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB18_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %edi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB18_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %x = or i32 %xx, 256
+  %shl = shl i32 %x, %y
+  %sub = sub i32 32, %y
+  %shr = lshr i32 %x, %sub
+  %z = or i32 %shr, %shl
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @rotl_maybe_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: rotl_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    roll %cl, %edi
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB19_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %edi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB19_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %shl = shl i32 %x, %y
+  %sub = sub i32 32, %y
+  %shr = lshr i32 %x, %sub
+  %z = or i32 %shr, %shl
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @rotl_with_fshl_known_nonzero(i32 %xx, i32 %y) {
+; CHECK-LABEL: rotl_with_fshl_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    orl $256, %edi # imm = 0x100
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    roll %cl, %edi
+; CHECK-NEXT:    bsfl %edi, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %x = or i32 %xx, 256
+  %z = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %y)
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @rotl_with_fshl_maybe_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: rotl_with_fshl_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    roll %cl, %edi
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB21_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %edi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB21_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %y)
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @sra_known_nonzero_sign_bit_set(i32 %x) {
+; CHECK-LABEL: sra_known_nonzero_sign_bit_set:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    movl $-2147360405, %eax # imm = 0x8001E16B
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    sarl %cl, %eax
+; CHECK-NEXT:    rep bsfl %eax, %eax
+; CHECK-NEXT:    retq
+  %z = ashr i32 2147606891, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @sra_known_nonzero_exact(i32 %x, i32 %yy) {
+; CHECK-LABEL: sra_known_nonzero_exact:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    orl $256, %esi # imm = 0x100
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    sarl %cl, %esi
+; CHECK-NEXT:    bsfl %esi, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %y = or i32 %yy, 256
+  %z = ashr exact i32 %y, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @sra_maybe_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: sra_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    sarl %cl, %esi
+; CHECK-NEXT:    testl %esi, %esi
+; CHECK-NEXT:    je .LBB24_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %esi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB24_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = ashr exact i32 %y, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @srl_known_nonzero_sign_bit_set(i32 %x) {
+; CHECK-LABEL: srl_known_nonzero_sign_bit_set:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    movl $-2147360405, %eax # imm = 0x8001E16B
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shrl %cl, %eax
+; CHECK-NEXT:    bsfl %eax, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %z = lshr i32 2147606891, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @srl_known_nonzero_exact(i32 %x, i32 %yy) {
+; CHECK-LABEL: srl_known_nonzero_exact:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    orl $256, %esi # imm = 0x100
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shrl %cl, %esi
+; CHECK-NEXT:    bsfl %esi, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %y = or i32 %yy, 256
+  %z = lshr exact i32 %y, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @srl_maybe_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: srl_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shrl %cl, %esi
+; CHECK-NEXT:    testl %esi, %esi
+; CHECK-NEXT:    je .LBB27_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %esi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB27_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = lshr exact i32 %y, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @udiv_known_nonzero(i32 %xx, i32 %y) {
+; CHECK-LABEL: udiv_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    orl $64, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %esi
+; CHECK-NEXT:    bsfl %eax, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %x = or i32 %xx, 64
+  %z = udiv exact i32 %x, %y
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @udiv_maybe_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: udiv_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %esi
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    je .LBB29_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %eax, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB29_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = udiv exact i32 %x, %y
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @sdiv_known_nonzero(i32 %xx, i32 %y) {
+; CHECK-LABEL: sdiv_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    orl $64, %eax
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %esi
+; CHECK-NEXT:    bsfl %eax, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %x = or i32 %xx, 64
+  %z = sdiv exact i32 %x, %y
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @sdiv_maybe_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: sdiv_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %esi
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    je .LBB31_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %eax, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB31_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = sdiv exact i32 %x, %y
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @add_known_nonzero(i32 %xx, i32 %y) {
+; CHECK-LABEL: add_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    orl $1, %edi
+; CHECK-NEXT:    addl %esi, %edi
+; CHECK-NEXT:    bsfl %edi, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %x = or i32 %xx, 1
+  %z = add nuw i32 %x, %y
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @add_maybe_zero(i32 %xx, i32 %y) {
+; CHECK-LABEL: add_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    orl $1, %edi
+; CHECK-NEXT:    addl %esi, %edi
+; CHECK-NEXT:    je .LBB33_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %edi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB33_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %x = or i32 %xx, 1
+  %z = add nsw i32 %x, %y
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @sub_known_nonzero_neg_case(i32 %xx) {
+; CHECK-LABEL: sub_known_nonzero_neg_case:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    movl $256, %eax # imm = 0x100
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shll %cl, %eax
+; CHECK-NEXT:    negl %eax
+; CHECK-NEXT:    bsfl %eax, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %x = shl nuw nsw i32 256, %xx
+  %z = sub i32 0, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @sub_known_nonzero_ne_case(i32 %xx, i32 %yy) {
+; CHECK-LABEL: sub_known_nonzero_ne_case:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    orl $64, %eax
+; CHECK-NEXT:    andl $-65, %edi
+; CHECK-NEXT:    subl %eax, %edi
+; CHECK-NEXT:    bsfl %edi, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %x = or i32 %xx, 64
+  %y = and i32 %xx, -65
+  %z = sub i32 %y, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @sub_maybe_zero(i32 %x) {
+; CHECK-LABEL: sub_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    orl $64, %eax
+; CHECK-NEXT:    subl %edi, %eax
+; CHECK-NEXT:    je .LBB36_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %eax, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB36_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %y = or i32 %x, 64
+  %z = sub i32 %y, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @sub_maybe_zero2(i32 %x) {
+; CHECK-LABEL: sub_maybe_zero2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    negl %edi
+; CHECK-NEXT:    je .LBB37_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %edi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB37_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = sub i32 0, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @mul_known_nonzero_nsw(i32 %x, i32 %yy) {
+; CHECK-LABEL: mul_known_nonzero_nsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    orl $256, %esi # imm = 0x100
+; CHECK-NEXT:    imull %edi, %esi
+; CHECK-NEXT:    testl %esi, %esi
+; CHECK-NEXT:    je .LBB38_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %esi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB38_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %y = or i32 %yy, 256
+  %z = mul nsw i32 %y, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @mul_known_nonzero_nuw(i32 %x, i32 %yy) {
+; CHECK-LABEL: mul_known_nonzero_nuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    orl $256, %esi # imm = 0x100
+; CHECK-NEXT:    imull %edi, %esi
+; CHECK-NEXT:    testl %esi, %esi
+; CHECK-NEXT:    je .LBB39_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %esi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB39_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %y = or i32 %yy, 256
+  %z = mul nuw i32 %y, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @mul_maybe_zero(i32 %x, i32 %y) {
+; CHECK-LABEL: mul_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    imull %esi, %edi
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB40_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %edi, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB40_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = mul nuw nsw i32 %y, %x
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @bitcast_known_nonzero(<2 x i16> %xx) {
+; CHECK-LABEL: bitcast_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; CHECK-NEXT:    pslld $23, %xmm0
+; CHECK-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT:    cvttps2dq %xmm0, %xmm0
+; CHECK-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; CHECK-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    bsfl %eax, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %x = shl nuw nsw <2 x i16> <i16 256, i16 256>, %xx
+  %z = bitcast <2 x i16> %x to i32
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @bitcast_maybe_zero(<2 x i16> %x) {
+; CHECK-LABEL: bitcast_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    je .LBB42_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %eax, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB42_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = bitcast <2 x i16> %x to i32
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @bitcast_from_float(float %x) {
+; CHECK-LABEL: bitcast_from_float:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    je .LBB43_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    rep bsfl %eax, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB43_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = bitcast float %x to i32
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @zext_known_nonzero(i16 %xx) {
+; CHECK-LABEL: zext_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    movl $256, %eax # imm = 0x100
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shll %cl, %eax
+; CHECK-NEXT:    movzwl %ax, %eax
+; CHECK-NEXT:    bsfl %eax, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %x = shl nuw nsw i16 256, %xx
+  %z = zext i16 %x to i32
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @zext_maybe_zero(i16 %x) {
+; CHECK-LABEL: zext_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    testw %di, %di
+; CHECK-NEXT:    je .LBB45_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    movzwl %di, %eax
+; CHECK-NEXT:    rep bsfl %eax, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB45_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = zext i16 %x to i32
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @sext_known_nonzero(i16 %xx) {
+; CHECK-LABEL: sext_known_nonzero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    movl $256, %eax # imm = 0x100
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shll %cl, %eax
+; CHECK-NEXT:    cwtl
+; CHECK-NEXT:    bsfl %eax, %ecx
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
+; CHECK-NEXT:    retq
+  %x = shl nuw nsw i16 256, %xx
+  %z = sext i16 %x to i32
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define i32 @sext_maybe_zero(i16 %x) {
+; CHECK-LABEL: sext_maybe_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    testw %di, %di
+; CHECK-NEXT:    je .LBB47_1
+; CHECK-NEXT:  # %bb.2: # %cond.false
+; CHECK-NEXT:    movswl %di, %eax
+; CHECK-NEXT:    rep bsfl %eax, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB47_1:
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    retq
+  %z = sext i16 %x to i32
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}