[X86] Add basic computeKnownBits support for X86ISD::BSR

[craig.topper]

Summary

The behavior is undefined for an input of 0, otherwise the result
is the position of the most significant set bit which must be in
the range [0, bitwidth-1]. So any bits above log2 of bitwidth
must be 0.

[RKSimon]

llvm/lib/Target/X86/X86ISelLowering.cpp

34045

I'm a bit worried about not handling the src==0 undef case - how well does this work if we guarded it with a KnownNeverZero check?

[RKSimon]

Also, can BSF be handled here as well?

[craig.topper]

In D89214#2325139, @RKSimon wrote:

Also, can BSF be handled here as well?

Technically yes, we just didn't have test coverage for it. Probably because we only use X86ISD::BSF when we immediately emit a CMOV for CTTZ and we need to connect the flag output. For CTTZ_ZERO_UNDEF we match BSF in isel patterns.

llvm/lib/Target/X86/X86ISelLowering.cpp

34045

The case I was trying to fix was this cross basic block case from the OPTIMIZE version of https://skanthak.homepage.t-online.de/llvm.html#case21

I doubt KnownNeverZero would work since its control flow dependent and it looks like the SelectionDAG implementation is only handles non-zero constants or an OR involving a non-zero constant.

[RKSimon]

llvm/lib/Target/X86/X86ISelLowering.cpp

34045

Sorry - I forgot about this! I think you're right in that in all the cases where X86ISD::BSR is generated, we have suitable zero-input handling in place - either we already treat this as CTLZ_ZERO_UNDEF (which computeknownbits already reports the same result for) or its part of an CTLZ and LowerCTLZ added zero-input wrapping already.

I'd feel better with a bit more of an explanation comment about our assumptions, but otherwise looks ok.

[RKSimon]

@craig.topper reverse ping

llvm/lib/Target/X86/X86ISelLowering.cpp

           Opc == ISD::INTRINSIC_W_CHAIN ||
           Opc == ISD::INTRINSIC_VOID) &&
          "Should use MaskedValueIsZero if you don't know whether Op"
          " is a target node!");
 
   Known.resetAll();
   switch (Opc) {
   default: break;
+  case X86ISD::BSR: {
+    // TODO: Bound with input known bits?
+    Known.Zero.setBitsFrom(Log2_32(BitWidth));
+    break;
+  }
   case X86ISD::SETCC:
     Known.Zero.setBitsFrom(1);
     break;
   case X86ISD::MOVMSK: {
     unsigned NumLoBits = Op.getOperand(0).getValueType().getVectorNumElements();
     Known.Zero.setBitsFrom(NumLoBits);
     break;
   }

llvm/test/CodeGen/X86/clz.ll

 ; X86-NOCMOV-LABEL: ctlz_i64:
 ; X86-NOCMOV:       # %bb.0:
 ; X86-NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NOCMOV-NEXT:    testl %eax, %eax
 ; X86-NOCMOV-NEXT:    jne .LBB7_1
 ; X86-NOCMOV-NEXT:  # %bb.2:
 ; X86-NOCMOV-NEXT:    bsrl {{[0-9]+}}(%esp), %eax
 ; X86-NOCMOV-NEXT:    xorl $31, %eax
-; X86-NOCMOV-NEXT:    addl $32, %eax
+; X86-NOCMOV-NEXT:    orl $32, %eax
 ; X86-NOCMOV-NEXT:    xorl %edx, %edx
 ; X86-NOCMOV-NEXT:    retl
 ; X86-NOCMOV-NEXT:  .LBB7_1:
 ; X86-NOCMOV-NEXT:    bsrl %eax, %eax
 ; X86-NOCMOV-NEXT:    xorl $31, %eax
 ; X86-NOCMOV-NEXT:    xorl %edx, %edx
 ; X86-NOCMOV-NEXT:    retl
 ;
 ; X86-CMOV-LABEL: ctlz_i64:
 ; X86-CMOV:       # %bb.0:
 ; X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-CMOV-NEXT:    bsrl %ecx, %edx
 ; X86-CMOV-NEXT:    xorl $31, %edx
 ; X86-CMOV-NEXT:    bsrl {{[0-9]+}}(%esp), %eax
 ; X86-CMOV-NEXT:    xorl $31, %eax
-; X86-CMOV-NEXT:    addl $32, %eax
+; X86-CMOV-NEXT:    orl $32, %eax
 ; X86-CMOV-NEXT:    testl %ecx, %ecx
 ; X86-CMOV-NEXT:    cmovnel %edx, %eax
 ; X86-CMOV-NEXT:    xorl %edx, %edx
 ; X86-CMOV-NEXT:    retl
 ;
 ; X64-LABEL: ctlz_i64:
 ; X64:       # %bb.0:
 ; X64-NEXT:    bsrq %rdi, %rax

llvm/test/CodeGen/X86/pr40090.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
 
 define i64 @foo(i64 %x, i64 %y) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    bsrq %rdi, %rax
-; CHECK-NEXT:    xorq $64, %rax
+; CHECK-NEXT:    orq $64, %rax
 ; CHECK-NEXT:    bsrq %rsi, %rcx
 ; CHECK-NEXT:    cmoveq %rax, %rcx
 ; CHECK-NEXT:    movl $63, %eax
 ; CHECK-NEXT:    subq %rcx, %rax
 ; CHECK-NEXT:    retq
   %1 = tail call i64 @llvm.ctlz.i64(i64 %x, i1 true)
   %2 = xor i64 %1, 127
   %3 = tail call i64 @llvm.ctlz.i64(i64 %y, i1 true)
   %4 = xor i64 %3, 63
   %5 = icmp eq i64 %y, 0
   %6 = select i1 %5, i64 %2, i64 %4
   %7 = sub nsw i64 63, %6
   ret i64 %7
 }
 
 declare i64 @llvm.ctlz.i64(i64, i1)

llvm/test/CodeGen/X86/scheduler-backtracking.ll

 ; ILP-NEXT:    andq %rcx, %r9
 ; ILP-NEXT:    bsrq %r9, %rcx
 ; ILP-NEXT:    xorq $63, %rcx
 ; ILP-NEXT:    andq %r8, %rdi
 ; ILP-NEXT:    bsrq %rdi, %r8
 ; ILP-NEXT:    andq %rdx, %r10
 ; ILP-NEXT:    bsrq %r10, %rdx
 ; ILP-NEXT:    xorq $63, %r8
-; ILP-NEXT:    addq $64, %rcx
+; ILP-NEXT:    orq $64, %rcx
 ; ILP-NEXT:    testq %rdi, %rdi
 ; ILP-NEXT:    movq $0, 24(%rax)
 ; ILP-NEXT:    movq $0, 16(%rax)
 ; ILP-NEXT:    movq $0, 8(%rax)
 ; ILP-NEXT:    cmovneq %r8, %rcx
 ; ILP-NEXT:    xorq $63, %rdx
 ; ILP-NEXT:    andq %rsi, %r11
 ; ILP-NEXT:    movl $127, %r8d

 ; HYBRID-NEXT:    sbbq %rcx, %rdi
 ; HYBRID-NEXT:    sbbq %r8, %r9
 ; HYBRID-NEXT:    andq %r8, %r9
 ; HYBRID-NEXT:    bsrq %r9, %r8
 ; HYBRID-NEXT:    xorq $63, %r8
 ; HYBRID-NEXT:    andq %rcx, %rdi
 ; HYBRID-NEXT:    bsrq %rdi, %rcx
 ; HYBRID-NEXT:    xorq $63, %rcx
-; HYBRID-NEXT:    addq $64, %rcx
+; HYBRID-NEXT:    orq $64, %rcx
 ; HYBRID-NEXT:    testq %r9, %r9
 ; HYBRID-NEXT:    cmovneq %r8, %rcx
 ; HYBRID-NEXT:    andq %rdx, %r10
 ; HYBRID-NEXT:    bsrq %r10, %rdx
 ; HYBRID-NEXT:    xorq $63, %rdx
 ; HYBRID-NEXT:    andq %rsi, %r11
 ; HYBRID-NEXT:    movl $127, %r8d
 ; HYBRID-NEXT:    bsrq %r11, %rsi

 ; BURR-NEXT:    sbbq %rcx, %rdi
 ; BURR-NEXT:    sbbq %r8, %r9
 ; BURR-NEXT:    andq %r8, %r9
 ; BURR-NEXT:    bsrq %r9, %r8
 ; BURR-NEXT:    xorq $63, %r8
 ; BURR-NEXT:    andq %rcx, %rdi
 ; BURR-NEXT:    bsrq %rdi, %rcx
 ; BURR-NEXT:    xorq $63, %rcx
-; BURR-NEXT:    addq $64, %rcx
+; BURR-NEXT:    orq $64, %rcx
 ; BURR-NEXT:    testq %r9, %r9
 ; BURR-NEXT:    cmovneq %r8, %rcx
 ; BURR-NEXT:    andq %rdx, %r10
 ; BURR-NEXT:    bsrq %r10, %rdx
 ; BURR-NEXT:    xorq $63, %rdx
 ; BURR-NEXT:    andq %rsi, %r11
 ; BURR-NEXT:    movl $127, %r8d
 ; BURR-NEXT:    bsrq %r11, %rsi

 ; SRC-NEXT:    andq %rdx, %r10
 ; SRC-NEXT:    andq %rcx, %r9
 ; SRC-NEXT:    andq %r8, %rdi
 ; SRC-NEXT:    andq %rsi, %r11
 ; SRC-NEXT:    bsrq %rdi, %rcx
 ; SRC-NEXT:    xorq $63, %rcx
 ; SRC-NEXT:    bsrq %r9, %rdx
 ; SRC-NEXT:    xorq $63, %rdx
-; SRC-NEXT:    addq $64, %rdx
+; SRC-NEXT:    orq $64, %rdx
 ; SRC-NEXT:    testq %rdi, %rdi
 ; SRC-NEXT:    cmovneq %rcx, %rdx
 ; SRC-NEXT:    bsrq %r10, %rcx
 ; SRC-NEXT:    xorq $63, %rcx
 ; SRC-NEXT:    bsrq %r11, %r8
 ; SRC-NEXT:    movl $127, %esi
 ; SRC-NEXT:    cmovneq %r8, %rsi
 ; SRC-NEXT:    xorq $63, %rsi

 ; LIN-NEXT:    testq %rsi, %rsi
 ; LIN-NEXT:    cmoveq %rdi, %rdx
 ; LIN-NEXT:    subq $-128, %rdx
 ; LIN-NEXT:    movl $0, %esi
 ; LIN-NEXT:    sbbq %rcx, %rsi
 ; LIN-NEXT:    andq %rcx, %rsi
 ; LIN-NEXT:    bsrq %rsi, %rcx
 ; LIN-NEXT:    xorq $63, %rcx
-; LIN-NEXT:    addq $64, %rcx
+; LIN-NEXT:    orq $64, %rcx
 ; LIN-NEXT:    sbbq %r8, %r9
 ; LIN-NEXT:    andq %r8, %r9
 ; LIN-NEXT:    bsrq %r9, %rdi
 ; LIN-NEXT:    xorq $63, %rdi
 ; LIN-NEXT:    testq %r9, %r9
 ; LIN-NEXT:    cmoveq %rcx, %rdi
 ; LIN-NEXT:    orq %rsi, %r9
 ; LIN-NEXT:    cmoveq %rdx, %rdi

 ; ILP-NEXT:    bsrq %rcx, %rdi
 ; ILP-NEXT:    notq %rdx
 ; ILP-NEXT:    andq %r11, %rdx
 ; ILP-NEXT:    xorq $63, %rdi
 ; ILP-NEXT:    notq %r8
 ; ILP-NEXT:    andq %r10, %r8
 ; ILP-NEXT:    bsrq %r8, %r10
 ; ILP-NEXT:    xorq $63, %r10
-; ILP-NEXT:    addq $64, %rdi
+; ILP-NEXT:    orq $64, %rdi
 ; ILP-NEXT:    bsrq %rdx, %r11
 ; ILP-NEXT:    notq %rsi
 ; ILP-NEXT:    testq %r8, %r8
 ; ILP-NEXT:    movq $0, 24(%rax)
 ; ILP-NEXT:    movq $0, 16(%rax)
 ; ILP-NEXT:    movq $0, 8(%rax)
 ; ILP-NEXT:    cmovneq %r10, %rdi
 ; ILP-NEXT:    xorq $63, %r11

 ; HYBRID-NEXT:    notq %r8
 ; HYBRID-NEXT:    andq %rdi, %r8
 ; HYBRID-NEXT:    bsrq %r8, %rbx
 ; HYBRID-NEXT:    xorq $63, %rbx
 ; HYBRID-NEXT:    notq %rcx
 ; HYBRID-NEXT:    andq %r11, %rcx
 ; HYBRID-NEXT:    bsrq %rcx, %rdi
 ; HYBRID-NEXT:    xorq $63, %rdi
-; HYBRID-NEXT:    addq $64, %rdi
+; HYBRID-NEXT:    orq $64, %rdi
 ; HYBRID-NEXT:    testq %r8, %r8
 ; HYBRID-NEXT:    cmovneq %rbx, %rdi
 ; HYBRID-NEXT:    notq %rdx
 ; HYBRID-NEXT:    andq %r10, %rdx
 ; HYBRID-NEXT:    bsrq %rdx, %rbx
 ; HYBRID-NEXT:    xorq $63, %rbx
 ; HYBRID-NEXT:    notq %rsi
 ; HYBRID-NEXT:    andq %r9, %rsi

 ; BURR-NEXT:    notq %r8
 ; BURR-NEXT:    andq %rdi, %r8
 ; BURR-NEXT:    bsrq %r8, %rbx
 ; BURR-NEXT:    xorq $63, %rbx
 ; BURR-NEXT:    notq %rcx
 ; BURR-NEXT:    andq %r11, %rcx
 ; BURR-NEXT:    bsrq %rcx, %rdi
 ; BURR-NEXT:    xorq $63, %rdi
-; BURR-NEXT:    addq $64, %rdi
+; BURR-NEXT:    orq $64, %rdi
 ; BURR-NEXT:    testq %r8, %r8
 ; BURR-NEXT:    cmovneq %rbx, %rdi
 ; BURR-NEXT:    notq %rdx
 ; BURR-NEXT:    andq %r10, %rdx
 ; BURR-NEXT:    bsrq %rdx, %rbx
 ; BURR-NEXT:    xorq $63, %rbx
 ; BURR-NEXT:    notq %rsi
 ; BURR-NEXT:    andq %r9, %rsi

 ; SRC-NEXT:    andq %r11, %rdx
 ; SRC-NEXT:    andq %rdi, %rcx
 ; SRC-NEXT:    andq %r10, %r8
 ; SRC-NEXT:    andq %r9, %rsi
 ; SRC-NEXT:    bsrq %r8, %r9
 ; SRC-NEXT:    xorq $63, %r9
 ; SRC-NEXT:    bsrq %rcx, %rdi
 ; SRC-NEXT:    xorq $63, %rdi
-; SRC-NEXT:    addq $64, %rdi
+; SRC-NEXT:    orq $64, %rdi
 ; SRC-NEXT:    testq %r8, %r8
 ; SRC-NEXT:    cmovneq %r9, %rdi
 ; SRC-NEXT:    bsrq %rdx, %r9
 ; SRC-NEXT:    xorq $63, %r9
 ; SRC-NEXT:    bsrq %rsi, %r10
 ; SRC-NEXT:    movl $127, %esi
 ; SRC-NEXT:    cmovneq %r10, %rsi
 ; SRC-NEXT:    xorq $63, %rsi

 ; LIN-NEXT:    cmoveq %rdi, %rsi
 ; LIN-NEXT:    subq $-128, %rsi
 ; LIN-NEXT:    movl $0, %edx
 ; LIN-NEXT:    sbbq %rcx, %rdx
 ; LIN-NEXT:    notq %rcx
 ; LIN-NEXT:    andq %rdx, %rcx
 ; LIN-NEXT:    bsrq %rcx, %rdx
 ; LIN-NEXT:    xorq $63, %rdx
-; LIN-NEXT:    addq $64, %rdx
+; LIN-NEXT:    orq $64, %rdx
 ; LIN-NEXT:    sbbq %r8, %r9
 ; LIN-NEXT:    notq %r8
 ; LIN-NEXT:    andq %r9, %r8
 ; LIN-NEXT:    bsrq %r8, %rdi
 ; LIN-NEXT:    xorq $63, %rdi
 ; LIN-NEXT:    testq %r8, %r8
 ; LIN-NEXT:    cmoveq %rdx, %rdi
 ; LIN-NEXT:    orq %rcx, %r8

 ; ILP-NEXT:    orq %rdx, %rsi
 ; ILP-NEXT:    je .LBB4_1
 ; ILP-NEXT:  # %bb.2: # %cond.false
 ; ILP-NEXT:    bsrq %r11, %rdx
 ; ILP-NEXT:    bsrq %rdi, %rcx
 ; ILP-NEXT:    xorq $63, %rcx
 ; ILP-NEXT:    bsrq %r10, %rsi
 ; ILP-NEXT:    xorq $63, %rsi
-; ILP-NEXT:    addq $64, %rsi
+; ILP-NEXT:    orq $64, %rsi
 ; ILP-NEXT:    testq %rdi, %rdi
 ; ILP-NEXT:    cmovneq %rcx, %rsi
 ; ILP-NEXT:    xorq $63, %rdx
 ; ILP-NEXT:    bsrq %rbx, %rcx
 ; ILP-NEXT:    xorq $63, %rcx
-; ILP-NEXT:    addq $64, %rcx
+; ILP-NEXT:    orq $64, %rcx
 ; ILP-NEXT:    testq %r11, %r11
 ; ILP-NEXT:    cmovneq %rdx, %rcx
-; ILP-NEXT:    subq $-128, %rcx
+; ILP-NEXT:    orq $128, %rcx
 ; ILP-NEXT:    xorl %r9d, %r9d
 ; ILP-NEXT:    orq %rdi, %r10
 ; ILP-NEXT:    cmovneq %rsi, %rcx
 ; ILP-NEXT:    jmp .LBB4_3
 ; ILP-NEXT:  .LBB4_1:
 ; ILP-NEXT:    movl $256, %ecx # imm = 0x100
 ; ILP-NEXT:  .LBB4_3: # %cond.end
 ; ILP-NEXT:    movq %rcx, (%rax)

 ; HYBRID-NEXT:    orq %rcx, %rsi
 ; HYBRID-NEXT:    orq %rdx, %rsi
 ; HYBRID-NEXT:    je .LBB4_1
 ; HYBRID-NEXT:  # %bb.2: # %cond.false
 ; HYBRID-NEXT:    bsrq %rdi, %rcx
 ; HYBRID-NEXT:    xorq $63, %rcx
 ; HYBRID-NEXT:    bsrq %r10, %rdx
 ; HYBRID-NEXT:    xorq $63, %rdx
-; HYBRID-NEXT:    addq $64, %rdx
+; HYBRID-NEXT:    orq $64, %rdx
 ; HYBRID-NEXT:    testq %rdi, %rdi
 ; HYBRID-NEXT:    cmovneq %rcx, %rdx
 ; HYBRID-NEXT:    bsrq %r11, %rsi
 ; HYBRID-NEXT:    xorq $63, %rsi
 ; HYBRID-NEXT:    bsrq %rbx, %rcx
 ; HYBRID-NEXT:    xorq $63, %rcx
-; HYBRID-NEXT:    addq $64, %rcx
+; HYBRID-NEXT:    orq $64, %rcx
 ; HYBRID-NEXT:    testq %r11, %r11
 ; HYBRID-NEXT:    cmovneq %rsi, %rcx
-; HYBRID-NEXT:    subq $-128, %rcx
+; HYBRID-NEXT:    orq $128, %rcx
 ; HYBRID-NEXT:    orq %rdi, %r10
 ; HYBRID-NEXT:    cmovneq %rdx, %rcx
 ; HYBRID-NEXT:    xorl %r9d, %r9d
 ; HYBRID-NEXT:    jmp .LBB4_3
 ; HYBRID-NEXT:  .LBB4_1:
 ; HYBRID-NEXT:    movl $256, %ecx # imm = 0x100
 ; HYBRID-NEXT:  .LBB4_3: # %cond.end
 ; HYBRID-NEXT:    movq %rcx, (%rax)

 ; BURR-NEXT:    orq %rcx, %rsi
 ; BURR-NEXT:    orq %rdx, %rsi
 ; BURR-NEXT:    je .LBB4_1
 ; BURR-NEXT:  # %bb.2: # %cond.false
 ; BURR-NEXT:    bsrq %rdi, %rcx
 ; BURR-NEXT:    xorq $63, %rcx
 ; BURR-NEXT:    bsrq %r10, %rdx
 ; BURR-NEXT:    xorq $63, %rdx
-; BURR-NEXT:    addq $64, %rdx
+; BURR-NEXT:    orq $64, %rdx
 ; BURR-NEXT:    testq %rdi, %rdi
 ; BURR-NEXT:    cmovneq %rcx, %rdx
 ; BURR-NEXT:    bsrq %r11, %rsi
 ; BURR-NEXT:    xorq $63, %rsi
 ; BURR-NEXT:    bsrq %rbx, %rcx
 ; BURR-NEXT:    xorq $63, %rcx
-; BURR-NEXT:    addq $64, %rcx
+; BURR-NEXT:    orq $64, %rcx
 ; BURR-NEXT:    testq %r11, %r11
 ; BURR-NEXT:    cmovneq %rsi, %rcx
-; BURR-NEXT:    subq $-128, %rcx
+; BURR-NEXT:    orq $128, %rcx
 ; BURR-NEXT:    orq %rdi, %r10
 ; BURR-NEXT:    cmovneq %rdx, %rcx
 ; BURR-NEXT:    xorl %r9d, %r9d
 ; BURR-NEXT:    jmp .LBB4_3
 ; BURR-NEXT:  .LBB4_1:
 ; BURR-NEXT:    movl $256, %ecx # imm = 0x100
 ; BURR-NEXT:  .LBB4_3: # %cond.end
 ; BURR-NEXT:    movq %rcx, (%rax)

 ; SRC-NEXT:    orq %rcx, %rsi
 ; SRC-NEXT:    orq %rdx, %rsi
 ; SRC-NEXT:    je .LBB4_1
 ; SRC-NEXT:  # %bb.2: # %cond.false
 ; SRC-NEXT:    bsrq %rdi, %rcx
 ; SRC-NEXT:    xorq $63, %rcx
 ; SRC-NEXT:    bsrq %r10, %rdx
 ; SRC-NEXT:    xorq $63, %rdx
-; SRC-NEXT:    addq $64, %rdx
+; SRC-NEXT:    orq $64, %rdx
 ; SRC-NEXT:    testq %rdi, %rdi
 ; SRC-NEXT:    cmovneq %rcx, %rdx
 ; SRC-NEXT:    bsrq %r11, %rsi
 ; SRC-NEXT:    xorq $63, %rsi
 ; SRC-NEXT:    bsrq %rbx, %rcx
 ; SRC-NEXT:    xorq $63, %rcx
-; SRC-NEXT:    addq $64, %rcx
+; SRC-NEXT:    orq $64, %rcx
 ; SRC-NEXT:    testq %r11, %r11
 ; SRC-NEXT:    cmovneq %rsi, %rcx
-; SRC-NEXT:    subq $-128, %rcx
+; SRC-NEXT:    orq $128, %rcx
 ; SRC-NEXT:    orq %rdi, %r10
 ; SRC-NEXT:    cmovneq %rdx, %rcx
 ; SRC-NEXT:    xorl %r9d, %r9d
 ; SRC-NEXT:    jmp .LBB4_3
 ; SRC-NEXT:  .LBB4_1:
 ; SRC-NEXT:    movl $256, %ecx # imm = 0x100
 ; SRC-NEXT:  .LBB4_3: # %cond.end
 ; SRC-NEXT:    movq %rcx, (%rax)

 ; LIN-NEXT:    sbbq %r8, %r11
 ; LIN-NEXT:    orq %rcx, %rsi
 ; LIN-NEXT:    orq %r8, %rdx
 ; LIN-NEXT:    orq %rsi, %rdx
 ; LIN-NEXT:    je .LBB4_1
 ; LIN-NEXT:  # %bb.2: # %cond.false
 ; LIN-NEXT:    bsrq %rbx, %rcx
 ; LIN-NEXT:    xorq $63, %rcx
-; LIN-NEXT:    addq $64, %rcx
+; LIN-NEXT:    orq $64, %rcx
 ; LIN-NEXT:    bsrq %rdi, %rdx
 ; LIN-NEXT:    xorq $63, %rdx
 ; LIN-NEXT:    testq %rdi, %rdi
 ; LIN-NEXT:    cmoveq %rcx, %rdx
-; LIN-NEXT:    subq $-128, %rdx
+; LIN-NEXT:    orq $128, %rdx
 ; LIN-NEXT:    bsrq %r10, %rsi
 ; LIN-NEXT:    xorq $63, %rsi
-; LIN-NEXT:    addq $64, %rsi
+; LIN-NEXT:    orq $64, %rsi
 ; LIN-NEXT:    bsrq %r11, %rcx
 ; LIN-NEXT:    xorq $63, %rcx
 ; LIN-NEXT:    testq %r11, %r11
 ; LIN-NEXT:    cmoveq %rsi, %rcx
 ; LIN-NEXT:    orq %r11, %r10
 ; LIN-NEXT:    cmoveq %rdx, %rcx
 ; LIN-NEXT:    xorl %r9d, %r9d
 ; LIN-NEXT:    jmp .LBB4_3