diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -3504,9 +3504,11 @@
 //   b) x & ~(-1 << nbits)
 //   c) x &  (-1 >> (32 - y))
 //   d) x << (32 - y) >> (32 - y)
+//   e) (1 << nbits) - 1
 bool X86DAGToDAGISel::matchBitExtract(SDNode *Node) {
   assert(
-      (Node->getOpcode() == ISD::AND || Node->getOpcode() == ISD::SRL) &&
+      (Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::AND ||
+       Node->getOpcode() == ISD::SRL) &&
       "Should be either an and-mask, or right-shift after clearing high bits.");
 
   // BEXTR is BMI instruction, BZHI is BMI2 instruction. We need at least one.
@@ -3692,6 +3694,8 @@
       if (!matchLowBitMask(Mask))
         return false;
     }
+  } else if (matchLowBitMask(SDValue(Node, 0))) {
+    X = CurDAG->getAllOnesConstant(SDLoc(Node), NVT);
   } else if (!matchPatternD(Node))
     return false;
 
@@ -5067,6 +5071,9 @@
 
     [[fallthrough]];
   case ISD::ADD:
+    if (Opcode == ISD::ADD && matchBitExtract(Node))
+      return;
+    [[fallthrough]];
   case ISD::SUB: {
     // Try to avoid folding immediates with multiple uses for optsize.
     // This code tries to select to register form directly to avoid going
diff --git a/llvm/test/CodeGen/X86/extract-bits.ll b/llvm/test/CodeGen/X86/extract-bits.ll
--- a/llvm/test/CodeGen/X86/extract-bits.ll
+++ b/llvm/test/CodeGen/X86/extract-bits.ll
@@ -4068,10 +4068,8 @@
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ebx
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    shrxl %eax, {{[0-9]+}}(%esp), %esi
-; X86-BMI2-NEXT:    movl %ebx, %eax
-; X86-BMI2-NEXT:    negb %al
-; X86-BMI2-NEXT:    movl $-1, %ecx
-; X86-BMI2-NEXT:    shrxl %eax, %ecx, %eax
+; X86-BMI2-NEXT:    movl $-1, %eax
+; X86-BMI2-NEXT:    bzhil %ebx, %eax, %eax
 ; X86-BMI2-NEXT:    movl %eax, (%esp)
 ; X86-BMI2-NEXT:    calll use32@PLT
 ; X86-BMI2-NEXT:    bzhil %ebx, %esi, %eax
@@ -4131,10 +4129,8 @@
 ; X64-BMI2-NEXT:    pushq %rax
 ; X64-BMI2-NEXT:    movl %edx, %ebx
 ; X64-BMI2-NEXT:    shrxl %esi, %edi, %ebp
-; X64-BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI2-NEXT:    negb %al
-; X64-BMI2-NEXT:    movl $-1, %ecx
-; X64-BMI2-NEXT:    shrxl %eax, %ecx, %edi
+; X64-BMI2-NEXT:    movl $-1, %eax
+; X64-BMI2-NEXT:    bzhil %edx, %eax, %edi
 ; X64-BMI2-NEXT:    callq use32@PLT
 ; X64-BMI2-NEXT:    bzhil %ebx, %ebp, %eax
 ; X64-BMI2-NEXT:    addq $8, %rsp
@@ -4202,10 +4198,8 @@
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ebx
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    shrxl %eax, {{[0-9]+}}(%esp), %esi
-; X86-BMI2-NEXT:    movl %ebx, %eax
-; X86-BMI2-NEXT:    negb %al
-; X86-BMI2-NEXT:    movl $-1, %ecx
-; X86-BMI2-NEXT:    shrxl %eax, %ecx, %eax
+; X86-BMI2-NEXT:    movl $-1, %eax
+; X86-BMI2-NEXT:    bzhil %ebx, %eax, %eax
 ; X86-BMI2-NEXT:    movl %eax, (%esp)
 ; X86-BMI2-NEXT:    calll use32@PLT
 ; X86-BMI2-NEXT:    bzhil %ebx, %esi, %eax
@@ -4265,10 +4259,8 @@
 ; X64-BMI2-NEXT:    pushq %rax
 ; X64-BMI2-NEXT:    movl %edx, %ebx
 ; X64-BMI2-NEXT:    shrxl %esi, %edi, %ebp
-; X64-BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI2-NEXT:    negb %al
-; X64-BMI2-NEXT:    movl $-1, %ecx
-; X64-BMI2-NEXT:    shrxl %eax, %ecx, %edi
+; X64-BMI2-NEXT:    movl $-1, %eax
+; X64-BMI2-NEXT:    bzhil %edx, %eax, %edi
 ; X64-BMI2-NEXT:    callq use32@PLT
 ; X64-BMI2-NEXT:    bzhil %ebx, %ebp, %eax
 ; X64-BMI2-NEXT:    addq $8, %rsp
@@ -4341,10 +4333,8 @@
 ; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-BMI2-NEXT:    shrxl %ecx, (%eax), %esi
-; X86-BMI2-NEXT:    movl %ebx, %eax
-; X86-BMI2-NEXT:    negb %al
-; X86-BMI2-NEXT:    movl $-1, %ecx
-; X86-BMI2-NEXT:    shrxl %eax, %ecx, %eax
+; X86-BMI2-NEXT:    movl $-1, %eax
+; X86-BMI2-NEXT:    bzhil %ebx, %eax, %eax
 ; X86-BMI2-NEXT:    movl %eax, (%esp)
 ; X86-BMI2-NEXT:    calll use32@PLT
 ; X86-BMI2-NEXT:    bzhil %ebx, %esi, %eax
@@ -4404,10 +4394,8 @@
 ; X64-BMI2-NEXT:    pushq %rax
 ; X64-BMI2-NEXT:    movl %edx, %ebx
 ; X64-BMI2-NEXT:    shrxl %esi, (%rdi), %ebp
-; X64-BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI2-NEXT:    negb %al
-; X64-BMI2-NEXT:    movl $-1, %ecx
-; X64-BMI2-NEXT:    shrxl %eax, %ecx, %edi
+; X64-BMI2-NEXT:    movl $-1, %eax
+; X64-BMI2-NEXT:    bzhil %edx, %eax, %edi
 ; X64-BMI2-NEXT:    callq use32@PLT
 ; X64-BMI2-NEXT:    bzhil %ebx, %ebp, %eax
 ; X64-BMI2-NEXT:    addq $8, %rsp
@@ -4479,10 +4467,8 @@
 ; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-BMI2-NEXT:    shrxl %ecx, (%eax), %esi
-; X86-BMI2-NEXT:    movl %ebx, %eax
-; X86-BMI2-NEXT:    negb %al
-; X86-BMI2-NEXT:    movl $-1, %ecx
-; X86-BMI2-NEXT:    shrxl %eax, %ecx, %eax
+; X86-BMI2-NEXT:    movl $-1, %eax
+; X86-BMI2-NEXT:    bzhil %ebx, %eax, %eax
 ; X86-BMI2-NEXT:    movl %eax, (%esp)
 ; X86-BMI2-NEXT:    calll use32@PLT
 ; X86-BMI2-NEXT:    bzhil %ebx, %esi, %eax
@@ -4542,10 +4528,8 @@
 ; X64-BMI2-NEXT:    pushq %rax
 ; X64-BMI2-NEXT:    movl %edx, %ebx
 ; X64-BMI2-NEXT:    shrxl %esi, (%rdi), %ebp
-; X64-BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI2-NEXT:    negb %al
-; X64-BMI2-NEXT:    movl $-1, %ecx
-; X64-BMI2-NEXT:    shrxl %eax, %ecx, %edi
+; X64-BMI2-NEXT:    movl $-1, %eax
+; X64-BMI2-NEXT:    bzhil %edx, %eax, %edi
 ; X64-BMI2-NEXT:    callq use32@PLT
 ; X64-BMI2-NEXT:    bzhil %ebx, %ebp, %eax
 ; X64-BMI2-NEXT:    addq $8, %rsp
@@ -4616,10 +4600,8 @@
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ebx
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    shrxl %eax, {{[0-9]+}}(%esp), %esi
-; X86-BMI2-NEXT:    movl %ebx, %eax
-; X86-BMI2-NEXT:    negb %al
-; X86-BMI2-NEXT:    movl $-1, %ecx
-; X86-BMI2-NEXT:    shrxl %eax, %ecx, %eax
+; X86-BMI2-NEXT:    movl $-1, %eax
+; X86-BMI2-NEXT:    bzhil %ebx, %eax, %eax
 ; X86-BMI2-NEXT:    movl %eax, (%esp)
 ; X86-BMI2-NEXT:    calll use32@PLT
 ; X86-BMI2-NEXT:    bzhil %ebx, %esi, %eax
@@ -4679,10 +4661,8 @@
 ; X64-BMI2-NEXT:    pushq %rax
 ; X64-BMI2-NEXT:    movl %edx, %ebx
 ; X64-BMI2-NEXT:    shrxl %esi, %edi, %ebp
-; X64-BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI2-NEXT:    negb %al
-; X64-BMI2-NEXT:    movl $-1, %ecx
-; X64-BMI2-NEXT:    shrxl %eax, %ecx, %edi
+; X64-BMI2-NEXT:    movl $-1, %eax
+; X64-BMI2-NEXT:    bzhil %edx, %eax, %edi
 ; X64-BMI2-NEXT:    callq use32@PLT
 ; X64-BMI2-NEXT:    bzhil %ebx, %ebp, %eax
 ; X64-BMI2-NEXT:    addq $8, %rsp
@@ -4761,10 +4741,8 @@
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ebx
 ; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-BMI2-NEXT:    shrxl %edi, {{[0-9]+}}(%esp), %esi
-; X86-BMI2-NEXT:    movl %ebx, %eax
-; X86-BMI2-NEXT:    negb %al
-; X86-BMI2-NEXT:    movl $-1, %ecx
-; X86-BMI2-NEXT:    shrxl %eax, %ecx, %eax
+; X86-BMI2-NEXT:    movl $-1, %eax
+; X86-BMI2-NEXT:    bzhil %ebx, %eax, %eax
 ; X86-BMI2-NEXT:    movl %eax, (%esp)
 ; X86-BMI2-NEXT:    calll use32@PLT
 ; X86-BMI2-NEXT:    bzhil %ebx, %esi, %esi
@@ -4833,10 +4811,8 @@
 ; X64-BMI2-NEXT:    movl %edx, %ebx
 ; X64-BMI2-NEXT:    movl %esi, %ebp
 ; X64-BMI2-NEXT:    shrxl %esi, %edi, %r14d
-; X64-BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI2-NEXT:    negb %al
-; X64-BMI2-NEXT:    movl $-1, %ecx
-; X64-BMI2-NEXT:    shrxl %eax, %ecx, %edi
+; X64-BMI2-NEXT:    movl $-1, %eax
+; X64-BMI2-NEXT:    bzhil %edx, %eax, %edi
 ; X64-BMI2-NEXT:    callq use32@PLT
 ; X64-BMI2-NEXT:    bzhil %ebx, %r14d, %ebx
 ; X64-BMI2-NEXT:    movl %ebp, %edi
@@ -5045,10 +5021,8 @@
 ; X64-BMI2-NEXT:    pushq %rax
 ; X64-BMI2-NEXT:    movq %rdx, %rbx
 ; X64-BMI2-NEXT:    shrxq %rsi, %rdi, %r14
-; X64-BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI2-NEXT:    negb %al
-; X64-BMI2-NEXT:    movq $-1, %rcx
-; X64-BMI2-NEXT:    shrxq %rax, %rcx, %rdi
+; X64-BMI2-NEXT:    movq $-1, %rax
+; X64-BMI2-NEXT:    bzhiq %rdx, %rax, %rdi
 ; X64-BMI2-NEXT:    callq use64@PLT
 ; X64-BMI2-NEXT:    bzhiq %rbx, %r14, %rax
 ; X64-BMI2-NEXT:    addq $8, %rsp
@@ -5252,10 +5226,8 @@
 ; X64-BMI2-NEXT:    movl %edx, %ebx
 ; X64-BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
 ; X64-BMI2-NEXT:    shrxq %rsi, %rdi, %r14
-; X64-BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI2-NEXT:    negb %al
-; X64-BMI2-NEXT:    movq $-1, %rcx
-; X64-BMI2-NEXT:    shrxq %rax, %rcx, %rdi
+; X64-BMI2-NEXT:    movq $-1, %rax
+; X64-BMI2-NEXT:    bzhiq %rbx, %rax, %rdi
 ; X64-BMI2-NEXT:    callq use64@PLT
 ; X64-BMI2-NEXT:    bzhiq %rbx, %r14, %rax
 ; X64-BMI2-NEXT:    addq $8, %rsp
@@ -5463,10 +5435,8 @@
 ; X64-BMI2-NEXT:    pushq %rax
 ; X64-BMI2-NEXT:    movq %rdx, %rbx
 ; X64-BMI2-NEXT:    shrxq %rsi, (%rdi), %r14
-; X64-BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI2-NEXT:    negb %al
-; X64-BMI2-NEXT:    movq $-1, %rcx
-; X64-BMI2-NEXT:    shrxq %rax, %rcx, %rdi
+; X64-BMI2-NEXT:    movq $-1, %rax
+; X64-BMI2-NEXT:    bzhiq %rdx, %rax, %rdi
 ; X64-BMI2-NEXT:    callq use64@PLT
 ; X64-BMI2-NEXT:    bzhiq %rbx, %r14, %rax
 ; X64-BMI2-NEXT:    addq $8, %rsp
@@ -5674,10 +5644,8 @@
 ; X64-BMI2-NEXT:    movl %edx, %ebx
 ; X64-BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
 ; X64-BMI2-NEXT:    shrxq %rsi, (%rdi), %r14
-; X64-BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI2-NEXT:    negb %al
-; X64-BMI2-NEXT:    movq $-1, %rcx
-; X64-BMI2-NEXT:    shrxq %rax, %rcx, %rdi
+; X64-BMI2-NEXT:    movq $-1, %rax
+; X64-BMI2-NEXT:    bzhiq %rbx, %rax, %rdi
 ; X64-BMI2-NEXT:    callq use64@PLT
 ; X64-BMI2-NEXT:    bzhiq %rbx, %r14, %rax
 ; X64-BMI2-NEXT:    addq $8, %rsp
@@ -5883,10 +5851,8 @@
 ; X64-BMI2-NEXT:    pushq %rax
 ; X64-BMI2-NEXT:    movq %rdx, %rbx
 ; X64-BMI2-NEXT:    shrxq %rsi, %rdi, %r14
-; X64-BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI2-NEXT:    negb %al
-; X64-BMI2-NEXT:    movq $-1, %rcx
-; X64-BMI2-NEXT:    shrxq %rax, %rcx, %rdi
+; X64-BMI2-NEXT:    movq $-1, %rax
+; X64-BMI2-NEXT:    bzhiq %rdx, %rax, %rdi
 ; X64-BMI2-NEXT:    callq use64@PLT
 ; X64-BMI2-NEXT:    bzhiq %rbx, %r14, %rax
 ; X64-BMI2-NEXT:    addq $8, %rsp
@@ -6109,10 +6075,8 @@
 ; X64-BMI2-NEXT:    movq %rdx, %rbx
 ; X64-BMI2-NEXT:    movq %rsi, %r14
 ; X64-BMI2-NEXT:    shrxq %rsi, %rdi, %r15
-; X64-BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI2-NEXT:    negb %al
-; X64-BMI2-NEXT:    movq $-1, %rcx
-; X64-BMI2-NEXT:    shrxq %rax, %rcx, %rdi
+; X64-BMI2-NEXT:    movq $-1, %rax
+; X64-BMI2-NEXT:    bzhiq %rdx, %rax, %rdi
 ; X64-BMI2-NEXT:    callq use64@PLT
 ; X64-BMI2-NEXT:    bzhiq %rbx, %r15, %rbx
 ; X64-BMI2-NEXT:    movq %r14, %rdi
diff --git a/llvm/test/CodeGen/X86/extract-lowbits.ll b/llvm/test/CodeGen/X86/extract-lowbits.ll
--- a/llvm/test/CodeGen/X86/extract-lowbits.ll
+++ b/llvm/test/CodeGen/X86/extract-lowbits.ll
@@ -368,8 +368,9 @@
   ret i64 %masked
 }
 
-define i64 @bzhi64_a1_indexzext(i64 %val, i8 zeroext %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bzhi64_a1_indexzext:
+; Check that we don't throw away the vreg_width-1 mask if not using shifts
+define i64 @bzhi64_a0_masked(i64 %val, i64 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bzhi64_a0_masked:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-NOBMI-NEXT:    movl $1, %eax
@@ -388,7 +389,7 @@
 ; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1-LABEL: bzhi64_a1_indexzext:
+; X86-BMI1-LABEL: bzhi64_a0_masked:
 ; X86-BMI1:       # %bb.0:
 ; X86-BMI1-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-BMI1-NEXT:    movl $1, %eax
@@ -407,7 +408,7 @@
 ; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-BMI1-NEXT:    retl
 ;
-; X86-BMI2-LABEL: bzhi64_a1_indexzext:
+; X86-BMI2-LABEL: bzhi64_a0_masked:
 ; X86-BMI2:       # %bb.0:
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-BMI2-NEXT:    movl $1, %eax
@@ -426,6 +427,93 @@
 ; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    retl
 ;
+; X64-NOBMI-LABEL: bzhi64_a0_masked:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    movl $1, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    shlq %cl, %rax
+; X64-NOBMI-NEXT:    decq %rax
+; X64-NOBMI-NEXT:    andq %rdi, %rax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1-LABEL: bzhi64_a0_masked:
+; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    andb $63, %sil
+; X64-BMI1-NEXT:    shll $8, %esi
+; X64-BMI1-NEXT:    bextrq %rsi, %rdi, %rax
+; X64-BMI1-NEXT:    retq
+;
+; X64-BMI2-LABEL: bzhi64_a0_masked:
+; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    andb $63, %sil
+; X64-BMI2-NEXT:    bzhiq %rsi, %rdi, %rax
+; X64-BMI2-NEXT:    retq
+  %numlowbits.masked = and i64 %numlowbits, 63
+  %onebit = shl i64 1, %numlowbits.masked
+  %mask = add nsw i64 %onebit, -1
+  %masked = and i64 %mask, %val
+  ret i64 %masked
+}
+
+define i64 @bzhi64_a1_indexzext(i64 %val, i8 zeroext %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bzhi64_a1_indexzext:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl $1, %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:    shldl %cl, %eax, %edx
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    je .LBB7_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %eax, %edx
+; X86-NOBMI-NEXT:    xorl %eax, %eax
+; X86-NOBMI-NEXT:  .LBB7_2:
+; X86-NOBMI-NEXT:    addl $-1, %eax
+; X86-NOBMI-NEXT:    adcl $-1, %edx
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1-LABEL: bzhi64_a1_indexzext:
+; X86-BMI1:       # %bb.0:
+; X86-BMI1-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI1-NEXT:    movl $1, %eax
+; X86-BMI1-NEXT:    xorl %edx, %edx
+; X86-BMI1-NEXT:    shldl %cl, %eax, %edx
+; X86-BMI1-NEXT:    shll %cl, %eax
+; X86-BMI1-NEXT:    testb $32, %cl
+; X86-BMI1-NEXT:    je .LBB7_2
+; X86-BMI1-NEXT:  # %bb.1:
+; X86-BMI1-NEXT:    movl %eax, %edx
+; X86-BMI1-NEXT:    xorl %eax, %eax
+; X86-BMI1-NEXT:  .LBB7_2:
+; X86-BMI1-NEXT:    addl $-1, %eax
+; X86-BMI1-NEXT:    adcl $-1, %edx
+; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %edx
+; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    retl
+;
+; X86-BMI2-LABEL: bzhi64_a1_indexzext:
+; X86-BMI2:       # %bb.0:
+; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT:    movl $1, %eax
+; X86-BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI2-NEXT:    shldl %cl, %eax, %edx
+; X86-BMI2-NEXT:    shlxl %ecx, %eax, %eax
+; X86-BMI2-NEXT:    testb $32, %cl
+; X86-BMI2-NEXT:    je .LBB7_2
+; X86-BMI2-NEXT:  # %bb.1:
+; X86-BMI2-NEXT:    movl %eax, %edx
+; X86-BMI2-NEXT:    xorl %eax, %eax
+; X86-BMI2-NEXT:  .LBB7_2:
+; X86-BMI2-NEXT:    addl $-1, %eax
+; X86-BMI2-NEXT:    adcl $-1, %edx
+; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %edx
+; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    retl
+;
 ; X64-NOBMI-LABEL: bzhi64_a1_indexzext:
 ; X64-NOBMI:       # %bb.0:
 ; X64-NOBMI-NEXT:    movl %esi, %ecx
@@ -466,11 +554,11 @@
 ; X86-NOBMI-NEXT:    shldl %cl, %eax, %edx
 ; X86-NOBMI-NEXT:    shll %cl, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB7_2
+; X86-NOBMI-NEXT:    je .LBB8_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %eax, %edx
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
-; X86-NOBMI-NEXT:  .LBB7_2:
+; X86-NOBMI-NEXT:  .LBB8_2:
 ; X86-NOBMI-NEXT:    addl $-1, %eax
 ; X86-NOBMI-NEXT:    adcl $-1, %edx
 ; X86-NOBMI-NEXT:    andl 4(%esi), %edx
@@ -488,11 +576,11 @@
 ; X86-BMI1-NEXT:    shldl %cl, %eax, %edx
 ; X86-BMI1-NEXT:    shll %cl, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    je .LBB7_2
+; X86-BMI1-NEXT:    je .LBB8_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %eax, %edx
 ; X86-BMI1-NEXT:    xorl %eax, %eax
-; X86-BMI1-NEXT:  .LBB7_2:
+; X86-BMI1-NEXT:  .LBB8_2:
 ; X86-BMI1-NEXT:    addl $-1, %eax
 ; X86-BMI1-NEXT:    adcl $-1, %edx
 ; X86-BMI1-NEXT:    andl 4(%esi), %edx
@@ -510,11 +598,11 @@
 ; X86-BMI2-NEXT:    shldl %cl, %eax, %edx
 ; X86-BMI2-NEXT:    shlxl %ecx, %eax, %eax
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    je .LBB7_2
+; X86-BMI2-NEXT:    je .LBB8_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %eax, %edx
 ; X86-BMI2-NEXT:    xorl %eax, %eax
-; X86-BMI2-NEXT:  .LBB7_2:
+; X86-BMI2-NEXT:  .LBB8_2:
 ; X86-BMI2-NEXT:    addl $-1, %eax
 ; X86-BMI2-NEXT:    adcl $-1, %edx
 ; X86-BMI2-NEXT:    andl 4(%esi), %edx
@@ -560,11 +648,11 @@
 ; X86-NOBMI-NEXT:    shldl %cl, %eax, %edx
 ; X86-NOBMI-NEXT:    shll %cl, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB8_2
+; X86-NOBMI-NEXT:    je .LBB9_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %eax, %edx
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
-; X86-NOBMI-NEXT:  .LBB8_2:
+; X86-NOBMI-NEXT:  .LBB9_2:
 ; X86-NOBMI-NEXT:    addl $-1, %eax
 ; X86-NOBMI-NEXT:    adcl $-1, %edx
 ; X86-NOBMI-NEXT:    andl 4(%esi), %edx
@@ -582,11 +670,11 @@
 ; X86-BMI1-NEXT:    shldl %cl, %eax, %edx
 ; X86-BMI1-NEXT:    shll %cl, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    je .LBB8_2
+; X86-BMI1-NEXT:    je .LBB9_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %eax, %edx
 ; X86-BMI1-NEXT:    xorl %eax, %eax
-; X86-BMI1-NEXT:  .LBB8_2:
+; X86-BMI1-NEXT:  .LBB9_2:
 ; X86-BMI1-NEXT:    addl $-1, %eax
 ; X86-BMI1-NEXT:    adcl $-1, %edx
 ; X86-BMI1-NEXT:    andl 4(%esi), %edx
@@ -604,11 +692,11 @@
 ; X86-BMI2-NEXT:    shldl %cl, %eax, %edx
 ; X86-BMI2-NEXT:    shlxl %ecx, %eax, %eax
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    je .LBB8_2
+; X86-BMI2-NEXT:    je .LBB9_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %eax, %edx
 ; X86-BMI2-NEXT:    xorl %eax, %eax
-; X86-BMI2-NEXT:  .LBB8_2:
+; X86-BMI2-NEXT:  .LBB9_2:
 ; X86-BMI2-NEXT:    addl $-1, %eax
 ; X86-BMI2-NEXT:    adcl $-1, %edx
 ; X86-BMI2-NEXT:    andl 4(%esi), %edx
@@ -655,11 +743,11 @@
 ; X86-NOBMI-NEXT:    shldl %cl, %eax, %edx
 ; X86-NOBMI-NEXT:    shll %cl, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB9_2
+; X86-NOBMI-NEXT:    je .LBB10_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %eax, %edx
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
-; X86-NOBMI-NEXT:  .LBB9_2:
+; X86-NOBMI-NEXT:  .LBB10_2:
 ; X86-NOBMI-NEXT:    addl $-1, %eax
 ; X86-NOBMI-NEXT:    adcl $-1, %edx
 ; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %edx
@@ -674,11 +762,11 @@
 ; X86-BMI1-NEXT:    shldl %cl, %eax, %edx
 ; X86-BMI1-NEXT:    shll %cl, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    je .LBB9_2
+; X86-BMI1-NEXT:    je .LBB10_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %eax, %edx
 ; X86-BMI1-NEXT:    xorl %eax, %eax
-; X86-BMI1-NEXT:  .LBB9_2:
+; X86-BMI1-NEXT:  .LBB10_2:
 ; X86-BMI1-NEXT:    addl $-1, %eax
 ; X86-BMI1-NEXT:    adcl $-1, %edx
 ; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %edx
@@ -693,11 +781,11 @@
 ; X86-BMI2-NEXT:    shldl %cl, %eax, %edx
 ; X86-BMI2-NEXT:    shlxl %ecx, %eax, %eax
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    je .LBB9_2
+; X86-BMI2-NEXT:    je .LBB10_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %eax, %edx
 ; X86-BMI2-NEXT:    xorl %eax, %eax
-; X86-BMI2-NEXT:  .LBB9_2:
+; X86-BMI2-NEXT:  .LBB10_2:
 ; X86-BMI2-NEXT:    addl $-1, %eax
 ; X86-BMI2-NEXT:    adcl $-1, %edx
 ; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %edx
@@ -741,10 +829,10 @@
 ; X86-NOBMI-NEXT:    shll %cl, %edx
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB10_2
+; X86-NOBMI-NEXT:    jne .LBB11_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %edx, %eax
-; X86-NOBMI-NEXT:  .LBB10_2:
+; X86-NOBMI-NEXT:  .LBB11_2:
 ; X86-NOBMI-NEXT:    decl %eax
 ; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-NOBMI-NEXT:    retl
@@ -756,10 +844,10 @@
 ; X86-BMI1-NEXT:    shll %cl, %edx
 ; X86-BMI1-NEXT:    xorl %eax, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    jne .LBB10_2
+; X86-BMI1-NEXT:    jne .LBB11_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %edx, %eax
-; X86-BMI1-NEXT:  .LBB10_2:
+; X86-BMI1-NEXT:  .LBB11_2:
 ; X86-BMI1-NEXT:    decl %eax
 ; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-BMI1-NEXT:    retl
@@ -769,11 +857,11 @@
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-BMI2-NEXT:    xorl %eax, %eax
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    jne .LBB10_2
+; X86-BMI2-NEXT:    jne .LBB11_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl $1, %eax
 ; X86-BMI2-NEXT:    shlxl %ecx, %eax, %eax
-; X86-BMI2-NEXT:  .LBB10_2:
+; X86-BMI2-NEXT:  .LBB11_2:
 ; X86-BMI2-NEXT:    decl %eax
 ; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    retl
@@ -985,10 +1073,10 @@
 ; X86-NOBMI-NEXT:    shll %cl, %edx
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB14_2
+; X86-NOBMI-NEXT:    jne .LBB15_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %edx, %eax
-; X86-NOBMI-NEXT:  .LBB14_2:
+; X86-NOBMI-NEXT:  .LBB15_2:
 ; X86-NOBMI-NEXT:    decl %eax
 ; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-NOBMI-NEXT:    retl
@@ -1000,10 +1088,10 @@
 ; X86-BMI1-NEXT:    shll %cl, %edx
 ; X86-BMI1-NEXT:    xorl %eax, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    jne .LBB14_2
+; X86-BMI1-NEXT:    jne .LBB15_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %edx, %eax
-; X86-BMI1-NEXT:  .LBB14_2:
+; X86-BMI1-NEXT:  .LBB15_2:
 ; X86-BMI1-NEXT:    decl %eax
 ; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-BMI1-NEXT:    retl
@@ -1013,11 +1101,11 @@
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-BMI2-NEXT:    xorl %eax, %eax
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    jne .LBB14_2
+; X86-BMI2-NEXT:    jne .LBB15_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl $1, %eax
 ; X86-BMI2-NEXT:    shlxl %ecx, %eax, %eax
-; X86-BMI2-NEXT:  .LBB14_2:
+; X86-BMI2-NEXT:  .LBB15_2:
 ; X86-BMI2-NEXT:    decl %eax
 ; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    retl
@@ -1321,13 +1409,13 @@
 ; X86-NOBMI-NEXT:    shll %cl, %esi
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB20_1
+; X86-NOBMI-NEXT:    jne .LBB21_1
 ; X86-NOBMI-NEXT:  # %bb.2:
 ; X86-NOBMI-NEXT:    movl %esi, %eax
-; X86-NOBMI-NEXT:    jmp .LBB20_3
-; X86-NOBMI-NEXT:  .LBB20_1:
+; X86-NOBMI-NEXT:    jmp .LBB21_3
+; X86-NOBMI-NEXT:  .LBB21_1:
 ; X86-NOBMI-NEXT:    movl %esi, %edx
-; X86-NOBMI-NEXT:  .LBB20_3:
+; X86-NOBMI-NEXT:  .LBB21_3:
 ; X86-NOBMI-NEXT:    notl %edx
 ; X86-NOBMI-NEXT:    notl %eax
 ; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
@@ -1342,11 +1430,11 @@
 ; X86-BMI1-NEXT:    movl $-1, %eax
 ; X86-BMI1-NEXT:    shll %cl, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    je .LBB20_2
+; X86-BMI1-NEXT:    je .LBB21_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %eax, %edx
 ; X86-BMI1-NEXT:    xorl %eax, %eax
-; X86-BMI1-NEXT:  .LBB20_2:
+; X86-BMI1-NEXT:  .LBB21_2:
 ; X86-BMI1-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
 ; X86-BMI1-NEXT:    andnl {{[0-9]+}}(%esp), %edx, %edx
 ; X86-BMI1-NEXT:    retl
@@ -1357,11 +1445,11 @@
 ; X86-BMI2-NEXT:    movl $-1, %ecx
 ; X86-BMI2-NEXT:    shlxl %edx, %ecx, %eax
 ; X86-BMI2-NEXT:    testb $32, %dl
-; X86-BMI2-NEXT:    je .LBB20_2
+; X86-BMI2-NEXT:    je .LBB21_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %eax, %ecx
 ; X86-BMI2-NEXT:    xorl %eax, %eax
-; X86-BMI2-NEXT:  .LBB20_2:
+; X86-BMI2-NEXT:  .LBB21_2:
 ; X86-BMI2-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
 ; X86-BMI2-NEXT:    andnl {{[0-9]+}}(%esp), %ecx, %edx
 ; X86-BMI2-NEXT:    retl
@@ -1402,13 +1490,13 @@
 ; X86-NOBMI-NEXT:    shll %cl, %esi
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB21_1
+; X86-NOBMI-NEXT:    jne .LBB22_1
 ; X86-NOBMI-NEXT:  # %bb.2:
 ; X86-NOBMI-NEXT:    movl %esi, %eax
-; X86-NOBMI-NEXT:    jmp .LBB21_3
-; X86-NOBMI-NEXT:  .LBB21_1:
+; X86-NOBMI-NEXT:    jmp .LBB22_3
+; X86-NOBMI-NEXT:  .LBB22_1:
 ; X86-NOBMI-NEXT:    movl %esi, %edx
-; X86-NOBMI-NEXT:  .LBB21_3:
+; X86-NOBMI-NEXT:  .LBB22_3:
 ; X86-NOBMI-NEXT:    notl %edx
 ; X86-NOBMI-NEXT:    notl %eax
 ; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
@@ -1423,11 +1511,11 @@
 ; X86-BMI1-NEXT:    movl $-1, %eax
 ; X86-BMI1-NEXT:    shll %cl, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    je .LBB21_2
+; X86-BMI1-NEXT:    je .LBB22_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %eax, %edx
 ; X86-BMI1-NEXT:    xorl %eax, %eax
-; X86-BMI1-NEXT:  .LBB21_2:
+; X86-BMI1-NEXT:  .LBB22_2:
 ; X86-BMI1-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
 ; X86-BMI1-NEXT:    andnl {{[0-9]+}}(%esp), %edx, %edx
 ; X86-BMI1-NEXT:    retl
@@ -1438,11 +1526,11 @@
 ; X86-BMI2-NEXT:    movl $-1, %ecx
 ; X86-BMI2-NEXT:    shlxl %edx, %ecx, %eax
 ; X86-BMI2-NEXT:    testb $32, %dl
-; X86-BMI2-NEXT:    je .LBB21_2
+; X86-BMI2-NEXT:    je .LBB22_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %eax, %ecx
 ; X86-BMI2-NEXT:    xorl %eax, %eax
-; X86-BMI2-NEXT:  .LBB21_2:
+; X86-BMI2-NEXT:  .LBB22_2:
 ; X86-BMI2-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
 ; X86-BMI2-NEXT:    andnl {{[0-9]+}}(%esp), %ecx, %edx
 ; X86-BMI2-NEXT:    retl
@@ -1488,13 +1576,13 @@
 ; X86-NOBMI-NEXT:    shll %cl, %edi
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB22_1
+; X86-NOBMI-NEXT:    jne .LBB23_1
 ; X86-NOBMI-NEXT:  # %bb.2:
 ; X86-NOBMI-NEXT:    movl %edi, %eax
-; X86-NOBMI-NEXT:    jmp .LBB22_3
-; X86-NOBMI-NEXT:  .LBB22_1:
+; X86-NOBMI-NEXT:    jmp .LBB23_3
+; X86-NOBMI-NEXT:  .LBB23_1:
 ; X86-NOBMI-NEXT:    movl %edi, %edx
-; X86-NOBMI-NEXT:  .LBB22_3:
+; X86-NOBMI-NEXT:  .LBB23_3:
 ; X86-NOBMI-NEXT:    notl %edx
 ; X86-NOBMI-NEXT:    notl %eax
 ; X86-NOBMI-NEXT:    andl (%esi), %eax
@@ -1512,11 +1600,11 @@
 ; X86-BMI1-NEXT:    movl $-1, %eax
 ; X86-BMI1-NEXT:    shll %cl, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    je .LBB22_2
+; X86-BMI1-NEXT:    je .LBB23_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %eax, %esi
 ; X86-BMI1-NEXT:    xorl %eax, %eax
-; X86-BMI1-NEXT:  .LBB22_2:
+; X86-BMI1-NEXT:  .LBB23_2:
 ; X86-BMI1-NEXT:    andnl (%edx), %eax, %eax
 ; X86-BMI1-NEXT:    andnl 4(%edx), %esi, %edx
 ; X86-BMI1-NEXT:    popl %esi
@@ -1530,11 +1618,11 @@
 ; X86-BMI2-NEXT:    movl $-1, %edx
 ; X86-BMI2-NEXT:    shlxl %ebx, %edx, %eax
 ; X86-BMI2-NEXT:    testb $32, %bl
-; X86-BMI2-NEXT:    je .LBB22_2
+; X86-BMI2-NEXT:    je .LBB23_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %eax, %edx
 ; X86-BMI2-NEXT:    xorl %eax, %eax
-; X86-BMI2-NEXT:  .LBB22_2:
+; X86-BMI2-NEXT:  .LBB23_2:
 ; X86-BMI2-NEXT:    andnl (%ecx), %eax, %eax
 ; X86-BMI2-NEXT:    andnl 4(%ecx), %edx, %edx
 ; X86-BMI2-NEXT:    popl %ebx
@@ -1579,13 +1667,13 @@
 ; X86-NOBMI-NEXT:    shll %cl, %edi
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB23_1
+; X86-NOBMI-NEXT:    jne .LBB24_1
 ; X86-NOBMI-NEXT:  # %bb.2:
 ; X86-NOBMI-NEXT:    movl %edi, %eax
-; X86-NOBMI-NEXT:    jmp .LBB23_3
-; X86-NOBMI-NEXT:  .LBB23_1:
+; X86-NOBMI-NEXT:    jmp .LBB24_3
+; X86-NOBMI-NEXT:  .LBB24_1:
 ; X86-NOBMI-NEXT:    movl %edi, %edx
-; X86-NOBMI-NEXT:  .LBB23_3:
+; X86-NOBMI-NEXT:  .LBB24_3:
 ; X86-NOBMI-NEXT:    notl %edx
 ; X86-NOBMI-NEXT:    notl %eax
 ; X86-NOBMI-NEXT:    andl (%esi), %eax
@@ -1603,11 +1691,11 @@
 ; X86-BMI1-NEXT:    movl $-1, %eax
 ; X86-BMI1-NEXT:    shll %cl, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    je .LBB23_2
+; X86-BMI1-NEXT:    je .LBB24_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %eax, %esi
 ; X86-BMI1-NEXT:    xorl %eax, %eax
-; X86-BMI1-NEXT:  .LBB23_2:
+; X86-BMI1-NEXT:  .LBB24_2:
 ; X86-BMI1-NEXT:    andnl (%edx), %eax, %eax
 ; X86-BMI1-NEXT:    andnl 4(%edx), %esi, %edx
 ; X86-BMI1-NEXT:    popl %esi
@@ -1621,11 +1709,11 @@
 ; X86-BMI2-NEXT:    movl $-1, %edx
 ; X86-BMI2-NEXT:    shlxl %ebx, %edx, %eax
 ; X86-BMI2-NEXT:    testb $32, %bl
-; X86-BMI2-NEXT:    je .LBB23_2
+; X86-BMI2-NEXT:    je .LBB24_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %eax, %edx
 ; X86-BMI2-NEXT:    xorl %eax, %eax
-; X86-BMI2-NEXT:  .LBB23_2:
+; X86-BMI2-NEXT:  .LBB24_2:
 ; X86-BMI2-NEXT:    andnl (%ecx), %eax, %eax
 ; X86-BMI2-NEXT:    andnl 4(%ecx), %edx, %edx
 ; X86-BMI2-NEXT:    popl %ebx
@@ -1671,13 +1759,13 @@
 ; X86-NOBMI-NEXT:    shll %cl, %esi
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB24_1
+; X86-NOBMI-NEXT:    jne .LBB25_1
 ; X86-NOBMI-NEXT:  # %bb.2:
 ; X86-NOBMI-NEXT:    movl %esi, %eax
-; X86-NOBMI-NEXT:    jmp .LBB24_3
-; X86-NOBMI-NEXT:  .LBB24_1:
+; X86-NOBMI-NEXT:    jmp .LBB25_3
+; X86-NOBMI-NEXT:  .LBB25_1:
 ; X86-NOBMI-NEXT:    movl %esi, %edx
-; X86-NOBMI-NEXT:  .LBB24_3:
+; X86-NOBMI-NEXT:  .LBB25_3:
 ; X86-NOBMI-NEXT:    notl %edx
 ; X86-NOBMI-NEXT:    notl %eax
 ; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
@@ -1692,11 +1780,11 @@
 ; X86-BMI1-NEXT:    movl $-1, %eax
 ; X86-BMI1-NEXT:    shll %cl, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    je .LBB24_2
+; X86-BMI1-NEXT:    je .LBB25_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %eax, %edx
 ; X86-BMI1-NEXT:    xorl %eax, %eax
-; X86-BMI1-NEXT:  .LBB24_2:
+; X86-BMI1-NEXT:  .LBB25_2:
 ; X86-BMI1-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
 ; X86-BMI1-NEXT:    andnl {{[0-9]+}}(%esp), %edx, %edx
 ; X86-BMI1-NEXT:    retl
@@ -1707,11 +1795,11 @@
 ; X86-BMI2-NEXT:    movl $-1, %ecx
 ; X86-BMI2-NEXT:    shlxl %edx, %ecx, %eax
 ; X86-BMI2-NEXT:    testb $32, %dl
-; X86-BMI2-NEXT:    je .LBB24_2
+; X86-BMI2-NEXT:    je .LBB25_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %eax, %ecx
 ; X86-BMI2-NEXT:    xorl %eax, %eax
-; X86-BMI2-NEXT:  .LBB24_2:
+; X86-BMI2-NEXT:  .LBB25_2:
 ; X86-BMI2-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
 ; X86-BMI2-NEXT:    andnl {{[0-9]+}}(%esp), %ecx, %edx
 ; X86-BMI2-NEXT:    retl
@@ -1753,10 +1841,10 @@
 ; X86-NOBMI-NEXT:    shll %cl, %edx
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB25_2
+; X86-NOBMI-NEXT:    jne .LBB26_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %edx, %eax
-; X86-NOBMI-NEXT:  .LBB25_2:
+; X86-NOBMI-NEXT:  .LBB26_2:
 ; X86-NOBMI-NEXT:    notl %eax
 ; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-NOBMI-NEXT:    retl
@@ -1768,10 +1856,10 @@
 ; X86-BMI1-NEXT:    shll %cl, %eax
 ; X86-BMI1-NEXT:    xorl %edx, %edx
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    jne .LBB25_2
+; X86-BMI1-NEXT:    jne .LBB26_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %eax, %edx
-; X86-BMI1-NEXT:  .LBB25_2:
+; X86-BMI1-NEXT:  .LBB26_2:
 ; X86-BMI1-NEXT:    andnl {{[0-9]+}}(%esp), %edx, %eax
 ; X86-BMI1-NEXT:    retl
 ;
@@ -1780,11 +1868,11 @@
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    xorl %ecx, %ecx
 ; X86-BMI2-NEXT:    testb $32, %al
-; X86-BMI2-NEXT:    jne .LBB25_2
+; X86-BMI2-NEXT:    jne .LBB26_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl $-1, %ecx
 ; X86-BMI2-NEXT:    shlxl %eax, %ecx, %ecx
-; X86-BMI2-NEXT:  .LBB25_2:
+; X86-BMI2-NEXT:  .LBB26_2:
 ; X86-BMI2-NEXT:    andnl {{[0-9]+}}(%esp), %ecx, %eax
 ; X86-BMI2-NEXT:    retl
 ;
@@ -1933,10 +2021,10 @@
 ; X86-NOBMI-NEXT:    shll %cl, %edx
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB28_2
+; X86-NOBMI-NEXT:    jne .LBB29_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %edx, %eax
-; X86-NOBMI-NEXT:  .LBB28_2:
+; X86-NOBMI-NEXT:  .LBB29_2:
 ; X86-NOBMI-NEXT:    notl %eax
 ; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-NOBMI-NEXT:    retl
@@ -1948,10 +2036,10 @@
 ; X86-BMI1-NEXT:    shll %cl, %eax
 ; X86-BMI1-NEXT:    xorl %edx, %edx
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    jne .LBB28_2
+; X86-BMI1-NEXT:    jne .LBB29_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %eax, %edx
-; X86-BMI1-NEXT:  .LBB28_2:
+; X86-BMI1-NEXT:  .LBB29_2:
 ; X86-BMI1-NEXT:    andnl {{[0-9]+}}(%esp), %edx, %eax
 ; X86-BMI1-NEXT:    retl
 ;
@@ -1960,11 +2048,11 @@
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    xorl %ecx, %ecx
 ; X86-BMI2-NEXT:    testb $32, %al
-; X86-BMI2-NEXT:    jne .LBB28_2
+; X86-BMI2-NEXT:    jne .LBB29_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl $-1, %ecx
 ; X86-BMI2-NEXT:    shlxl %eax, %ecx, %ecx
-; X86-BMI2-NEXT:  .LBB28_2:
+; X86-BMI2-NEXT:  .LBB29_2:
 ; X86-BMI2-NEXT:    andnl {{[0-9]+}}(%esp), %ecx, %eax
 ; X86-BMI2-NEXT:    retl
 ;
@@ -2029,16 +2117,12 @@
 ;
 ; X86-BMI2-LABEL: bzhi32_c0:
 ; X86-BMI2:       # %bb.0:
-; X86-BMI2-NEXT:    pushl %esi
-; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
-; X86-BMI2-NEXT:    bzhil %edx, {{[0-9]+}}(%esp), %eax
-; X86-BMI2-NEXT:    # kill: def $dl killed $dl killed $edx def $edx
-; X86-BMI2-NEXT:    negb %dl
-; X86-BMI2-NEXT:    movl $-1, %esi
-; X86-BMI2-NEXT:    shrxl %edx, %esi, %edx
-; X86-BMI2-NEXT:    movl %edx, (%ecx)
-; X86-BMI2-NEXT:    popl %esi
+; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT:    movl $-1, %edx
+; X86-BMI2-NEXT:    bzhil %ecx, %edx, %edx
+; X86-BMI2-NEXT:    movl %edx, (%eax)
+; X86-BMI2-NEXT:    bzhil %ecx, {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-NOBMI-LABEL: bzhi32_c0:
@@ -2065,12 +2149,10 @@
 ;
 ; X64-BMI2-LABEL: bzhi32_c0:
 ; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    movl $-1, %eax
+; X64-BMI2-NEXT:    bzhil %esi, %eax, %eax
+; X64-BMI2-NEXT:    movl %eax, (%rdx)
 ; X64-BMI2-NEXT:    bzhil %esi, %edi, %eax
-; X64-BMI2-NEXT:    # kill: def $sil killed $sil killed $esi def $esi
-; X64-BMI2-NEXT:    negb %sil
-; X64-BMI2-NEXT:    movl $-1, %ecx
-; X64-BMI2-NEXT:    shrxl %esi, %ecx, %ecx
-; X64-BMI2-NEXT:    movl %ecx, (%rdx)
 ; X64-BMI2-NEXT:    retq
   %numhighbits = sub i32 32, %numlowbits
   %mask = lshr i32 -1, %numhighbits
@@ -2106,16 +2188,12 @@
 ;
 ; X86-BMI2-LABEL: bzhi32_c1_indexzext:
 ; X86-BMI2:       # %bb.0:
-; X86-BMI2-NEXT:    pushl %esi
-; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
-; X86-BMI2-NEXT:    bzhil %edx, {{[0-9]+}}(%esp), %eax
-; X86-BMI2-NEXT:    # kill: def $dl killed $dl killed $edx def $edx
-; X86-BMI2-NEXT:    negb %dl
-; X86-BMI2-NEXT:    movl $-1, %esi
-; X86-BMI2-NEXT:    shrxl %edx, %esi, %edx
-; X86-BMI2-NEXT:    movl %edx, (%ecx)
-; X86-BMI2-NEXT:    popl %esi
+; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT:    movl $-1, %edx
+; X86-BMI2-NEXT:    bzhil %ecx, %edx, %edx
+; X86-BMI2-NEXT:    movl %edx, (%eax)
+; X86-BMI2-NEXT:    bzhil %ecx, {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-NOBMI-LABEL: bzhi32_c1_indexzext:
@@ -2142,12 +2220,10 @@
 ;
 ; X64-BMI2-LABEL: bzhi32_c1_indexzext:
 ; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    movl $-1, %eax
+; X64-BMI2-NEXT:    bzhil %esi, %eax, %eax
+; X64-BMI2-NEXT:    movl %eax, (%rdx)
 ; X64-BMI2-NEXT:    bzhil %esi, %edi, %eax
-; X64-BMI2-NEXT:    # kill: def $sil killed $sil killed $esi def $esi
-; X64-BMI2-NEXT:    negb %sil
-; X64-BMI2-NEXT:    movl $-1, %ecx
-; X64-BMI2-NEXT:    shrxl %esi, %ecx, %ecx
-; X64-BMI2-NEXT:    movl %ecx, (%rdx)
 ; X64-BMI2-NEXT:    retq
   %numhighbits = sub i8 32, %numlowbits
   %sh_prom = zext i8 %numhighbits to i32
@@ -2196,12 +2272,10 @@
 ; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
-; X86-BMI2-NEXT:    bzhil %edx, (%eax), %eax
-; X86-BMI2-NEXT:    # kill: def $dl killed $dl killed $edx def $edx
-; X86-BMI2-NEXT:    negb %dl
 ; X86-BMI2-NEXT:    movl $-1, %esi
-; X86-BMI2-NEXT:    shrxl %edx, %esi, %edx
-; X86-BMI2-NEXT:    movl %edx, (%ecx)
+; X86-BMI2-NEXT:    bzhil %edx, %esi, %esi
+; X86-BMI2-NEXT:    bzhil %edx, (%eax), %eax
+; X86-BMI2-NEXT:    movl %esi, (%ecx)
 ; X86-BMI2-NEXT:    popl %esi
 ; X86-BMI2-NEXT:    retl
 ;
@@ -2231,11 +2305,9 @@
 ;
 ; X64-BMI2-LABEL: bzhi32_c2_load:
 ; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    movl $-1, %eax
+; X64-BMI2-NEXT:    bzhil %esi, %eax, %ecx
 ; X64-BMI2-NEXT:    bzhil %esi, (%rdi), %eax
-; X64-BMI2-NEXT:    # kill: def $sil killed $sil killed $esi def $esi
-; X64-BMI2-NEXT:    negb %sil
-; X64-BMI2-NEXT:    movl $-1, %ecx
-; X64-BMI2-NEXT:    shrxl %esi, %ecx, %ecx
 ; X64-BMI2-NEXT:    movl %ecx, (%rdx)
 ; X64-BMI2-NEXT:    retq
   %val = load i32, ptr %w
@@ -2285,12 +2357,10 @@
 ; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
-; X86-BMI2-NEXT:    bzhil %edx, (%eax), %eax
-; X86-BMI2-NEXT:    # kill: def $dl killed $dl killed $edx def $edx
-; X86-BMI2-NEXT:    negb %dl
 ; X86-BMI2-NEXT:    movl $-1, %esi
-; X86-BMI2-NEXT:    shrxl %edx, %esi, %edx
-; X86-BMI2-NEXT:    movl %edx, (%ecx)
+; X86-BMI2-NEXT:    bzhil %edx, %esi, %esi
+; X86-BMI2-NEXT:    bzhil %edx, (%eax), %eax
+; X86-BMI2-NEXT:    movl %esi, (%ecx)
 ; X86-BMI2-NEXT:    popl %esi
 ; X86-BMI2-NEXT:    retl
 ;
@@ -2320,11 +2390,9 @@
 ;
 ; X64-BMI2-LABEL: bzhi32_c3_load_indexzext:
 ; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    movl $-1, %eax
+; X64-BMI2-NEXT:    bzhil %esi, %eax, %ecx
 ; X64-BMI2-NEXT:    bzhil %esi, (%rdi), %eax
-; X64-BMI2-NEXT:    # kill: def $sil killed $sil killed $esi def $esi
-; X64-BMI2-NEXT:    negb %sil
-; X64-BMI2-NEXT:    movl $-1, %ecx
-; X64-BMI2-NEXT:    shrxl %esi, %ecx, %ecx
 ; X64-BMI2-NEXT:    movl %ecx, (%rdx)
 ; X64-BMI2-NEXT:    retq
   %val = load i32, ptr %w
@@ -2363,16 +2431,12 @@
 ;
 ; X86-BMI2-LABEL: bzhi32_c4_commutative:
 ; X86-BMI2:       # %bb.0:
-; X86-BMI2-NEXT:    pushl %esi
-; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
-; X86-BMI2-NEXT:    bzhil %edx, {{[0-9]+}}(%esp), %eax
-; X86-BMI2-NEXT:    # kill: def $dl killed $dl killed $edx def $edx
-; X86-BMI2-NEXT:    negb %dl
-; X86-BMI2-NEXT:    movl $-1, %esi
-; X86-BMI2-NEXT:    shrxl %edx, %esi, %edx
-; X86-BMI2-NEXT:    movl %edx, (%ecx)
-; X86-BMI2-NEXT:    popl %esi
+; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT:    movl $-1, %edx
+; X86-BMI2-NEXT:    bzhil %ecx, %edx, %edx
+; X86-BMI2-NEXT:    movl %edx, (%eax)
+; X86-BMI2-NEXT:    bzhil %ecx, {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-NOBMI-LABEL: bzhi32_c4_commutative:
@@ -2399,12 +2463,10 @@
 ;
 ; X64-BMI2-LABEL: bzhi32_c4_commutative:
 ; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    movl $-1, %eax
+; X64-BMI2-NEXT:    bzhil %esi, %eax, %eax
+; X64-BMI2-NEXT:    movl %eax, (%rdx)
 ; X64-BMI2-NEXT:    bzhil %esi, %edi, %eax
-; X64-BMI2-NEXT:    # kill: def $sil killed $sil killed $esi def $esi
-; X64-BMI2-NEXT:    negb %sil
-; X64-BMI2-NEXT:    movl $-1, %ecx
-; X64-BMI2-NEXT:    shrxl %esi, %ecx, %ecx
-; X64-BMI2-NEXT:    movl %ecx, (%rdx)
 ; X64-BMI2-NEXT:    retq
   %numhighbits = sub i32 32, %numlowbits
   %mask = lshr i32 -1, %numhighbits
@@ -2426,11 +2488,11 @@
 ; X86-NOBMI-NEXT:    movl $-1, %edx
 ; X86-NOBMI-NEXT:    shrl %cl, %edx
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB34_2
+; X86-NOBMI-NEXT:    je .LBB35_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %edx, %eax
 ; X86-NOBMI-NEXT:    xorl %edx, %edx
-; X86-NOBMI-NEXT:  .LBB34_2:
+; X86-NOBMI-NEXT:  .LBB35_2:
 ; X86-NOBMI-NEXT:    movl %edx, 4(%esi)
 ; X86-NOBMI-NEXT:    movl %eax, (%esi)
 ; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
@@ -2448,11 +2510,11 @@
 ; X86-BMI1-NEXT:    movl $-1, %edx
 ; X86-BMI1-NEXT:    shrl %cl, %edx
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    je .LBB34_2
+; X86-BMI1-NEXT:    je .LBB35_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %edx, %eax
 ; X86-BMI1-NEXT:    xorl %edx, %edx
-; X86-BMI1-NEXT:  .LBB34_2:
+; X86-BMI1-NEXT:  .LBB35_2:
 ; X86-BMI1-NEXT:    movl %edx, 4(%esi)
 ; X86-BMI1-NEXT:    movl %eax, (%esi)
 ; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %eax
@@ -2469,11 +2531,11 @@
 ; X86-BMI2-NEXT:    movl $-1, %eax
 ; X86-BMI2-NEXT:    shrxl %ebx, %eax, %edx
 ; X86-BMI2-NEXT:    testb $32, %bl
-; X86-BMI2-NEXT:    je .LBB34_2
+; X86-BMI2-NEXT:    je .LBB35_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %edx, %eax
 ; X86-BMI2-NEXT:    xorl %edx, %edx
-; X86-BMI2-NEXT:  .LBB34_2:
+; X86-BMI2-NEXT:  .LBB35_2:
 ; X86-BMI2-NEXT:    movl %edx, 4(%ecx)
 ; X86-BMI2-NEXT:    movl %eax, (%ecx)
 ; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %eax
@@ -2505,12 +2567,10 @@
 ;
 ; X64-BMI2-LABEL: bzhi64_c0:
 ; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    movq $-1, %rax
+; X64-BMI2-NEXT:    bzhiq %rsi, %rax, %rax
+; X64-BMI2-NEXT:    movq %rax, (%rdx)
 ; X64-BMI2-NEXT:    bzhiq %rsi, %rdi, %rax
-; X64-BMI2-NEXT:    # kill: def $sil killed $sil killed $rsi def $rsi
-; X64-BMI2-NEXT:    negb %sil
-; X64-BMI2-NEXT:    movq $-1, %rcx
-; X64-BMI2-NEXT:    shrxq %rsi, %rcx, %rcx
-; X64-BMI2-NEXT:    movq %rcx, (%rdx)
 ; X64-BMI2-NEXT:    retq
   %numhighbits = sub i64 64, %numlowbits
   %mask = lshr i64 -1, %numhighbits
@@ -2530,11 +2590,11 @@
 ; X86-NOBMI-NEXT:    movl $-1, %edx
 ; X86-NOBMI-NEXT:    shrl %cl, %edx
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB35_2
+; X86-NOBMI-NEXT:    je .LBB36_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %edx, %eax
 ; X86-NOBMI-NEXT:    xorl %edx, %edx
-; X86-NOBMI-NEXT:  .LBB35_2:
+; X86-NOBMI-NEXT:  .LBB36_2:
 ; X86-NOBMI-NEXT:    movl %edx, 4(%esi)
 ; X86-NOBMI-NEXT:    movl %eax, (%esi)
 ; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
@@ -2552,11 +2612,11 @@
 ; X86-BMI1-NEXT:    movl $-1, %edx
 ; X86-BMI1-NEXT:    shrl %cl, %edx
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    je .LBB35_2
+; X86-BMI1-NEXT:    je .LBB36_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %edx, %eax
 ; X86-BMI1-NEXT:    xorl %edx, %edx
-; X86-BMI1-NEXT:  .LBB35_2:
+; X86-BMI1-NEXT:  .LBB36_2:
 ; X86-BMI1-NEXT:    movl %edx, 4(%esi)
 ; X86-BMI1-NEXT:    movl %eax, (%esi)
 ; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %eax
@@ -2573,11 +2633,11 @@
 ; X86-BMI2-NEXT:    movl $-1, %eax
 ; X86-BMI2-NEXT:    shrxl %ebx, %eax, %edx
 ; X86-BMI2-NEXT:    testb $32, %bl
-; X86-BMI2-NEXT:    je .LBB35_2
+; X86-BMI2-NEXT:    je .LBB36_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %edx, %eax
 ; X86-BMI2-NEXT:    xorl %edx, %edx
-; X86-BMI2-NEXT:  .LBB35_2:
+; X86-BMI2-NEXT:  .LBB36_2:
 ; X86-BMI2-NEXT:    movl %edx, 4(%ecx)
 ; X86-BMI2-NEXT:    movl %eax, (%ecx)
 ; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %eax
@@ -2610,12 +2670,10 @@
 ; X64-BMI2-LABEL: bzhi64_c1_indexzext:
 ; X64-BMI2:       # %bb.0:
 ; X64-BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-BMI2-NEXT:    movq $-1, %rax
+; X64-BMI2-NEXT:    bzhiq %rsi, %rax, %rax
+; X64-BMI2-NEXT:    movq %rax, (%rdx)
 ; X64-BMI2-NEXT:    bzhiq %rsi, %rdi, %rax
-; X64-BMI2-NEXT:    # kill: def $sil killed $sil killed $rsi def $rsi
-; X64-BMI2-NEXT:    negb %sil
-; X64-BMI2-NEXT:    movq $-1, %rcx
-; X64-BMI2-NEXT:    shrxq %rsi, %rcx, %rcx
-; X64-BMI2-NEXT:    movq %rcx, (%rdx)
 ; X64-BMI2-NEXT:    retq
   %numhighbits = sub i8 64, %numlowbits
   %sh_prom = zext i8 %numhighbits to i64
@@ -2639,11 +2697,11 @@
 ; X86-NOBMI-NEXT:    movl $-1, %ebx
 ; X86-NOBMI-NEXT:    shrl %cl, %ebx
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB36_2
+; X86-NOBMI-NEXT:    je .LBB37_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %ebx, %edi
 ; X86-NOBMI-NEXT:    xorl %ebx, %ebx
-; X86-NOBMI-NEXT:  .LBB36_2:
+; X86-NOBMI-NEXT:  .LBB37_2:
 ; X86-NOBMI-NEXT:    movl 4(%eax), %edx
 ; X86-NOBMI-NEXT:    andl %ebx, %edx
 ; X86-NOBMI-NEXT:    movl (%eax), %eax
@@ -2668,11 +2726,11 @@
 ; X86-BMI1-NEXT:    movl $-1, %ebx
 ; X86-BMI1-NEXT:    shrl %cl, %ebx
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    je .LBB36_2
+; X86-BMI1-NEXT:    je .LBB37_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %ebx, %edi
 ; X86-BMI1-NEXT:    xorl %ebx, %ebx
-; X86-BMI1-NEXT:  .LBB36_2:
+; X86-BMI1-NEXT:  .LBB37_2:
 ; X86-BMI1-NEXT:    movl 4(%eax), %edx
 ; X86-BMI1-NEXT:    andl %ebx, %edx
 ; X86-BMI1-NEXT:    movl (%eax), %eax
@@ -2695,11 +2753,11 @@
 ; X86-BMI2-NEXT:    movl $-1, %esi
 ; X86-BMI2-NEXT:    shrxl %edx, %esi, %edi
 ; X86-BMI2-NEXT:    testb $32, %dl
-; X86-BMI2-NEXT:    je .LBB36_2
+; X86-BMI2-NEXT:    je .LBB37_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %edi, %esi
 ; X86-BMI2-NEXT:    xorl %edi, %edi
-; X86-BMI2-NEXT:  .LBB36_2:
+; X86-BMI2-NEXT:  .LBB37_2:
 ; X86-BMI2-NEXT:    movl 4(%eax), %edx
 ; X86-BMI2-NEXT:    andl %edi, %edx
 ; X86-BMI2-NEXT:    movl (%eax), %eax
@@ -2736,11 +2794,9 @@
 ;
 ; X64-BMI2-LABEL: bzhi64_c2_load:
 ; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    movq $-1, %rax
+; X64-BMI2-NEXT:    bzhiq %rsi, %rax, %rcx
 ; X64-BMI2-NEXT:    bzhiq %rsi, (%rdi), %rax
-; X64-BMI2-NEXT:    # kill: def $sil killed $sil killed $rsi def $rsi
-; X64-BMI2-NEXT:    negb %sil
-; X64-BMI2-NEXT:    movq $-1, %rcx
-; X64-BMI2-NEXT:    shrxq %rsi, %rcx, %rcx
 ; X64-BMI2-NEXT:    movq %rcx, (%rdx)
 ; X64-BMI2-NEXT:    retq
   %val = load i64, ptr %w
@@ -2765,11 +2821,11 @@
 ; X86-NOBMI-NEXT:    movl $-1, %ebx
 ; X86-NOBMI-NEXT:    shrl %cl, %ebx
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB37_2
+; X86-NOBMI-NEXT:    je .LBB38_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %ebx, %edi
 ; X86-NOBMI-NEXT:    xorl %ebx, %ebx
-; X86-NOBMI-NEXT:  .LBB37_2:
+; X86-NOBMI-NEXT:  .LBB38_2:
 ; X86-NOBMI-NEXT:    movl 4(%eax), %edx
 ; X86-NOBMI-NEXT:    andl %ebx, %edx
 ; X86-NOBMI-NEXT:    movl (%eax), %eax
@@ -2794,11 +2850,11 @@
 ; X86-BMI1-NEXT:    movl $-1, %ebx
 ; X86-BMI1-NEXT:    shrl %cl, %ebx
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    je .LBB37_2
+; X86-BMI1-NEXT:    je .LBB38_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %ebx, %edi
 ; X86-BMI1-NEXT:    xorl %ebx, %ebx
-; X86-BMI1-NEXT:  .LBB37_2:
+; X86-BMI1-NEXT:  .LBB38_2:
 ; X86-BMI1-NEXT:    movl 4(%eax), %edx
 ; X86-BMI1-NEXT:    andl %ebx, %edx
 ; X86-BMI1-NEXT:    movl (%eax), %eax
@@ -2821,11 +2877,11 @@
 ; X86-BMI2-NEXT:    movl $-1, %esi
 ; X86-BMI2-NEXT:    shrxl %edx, %esi, %edi
 ; X86-BMI2-NEXT:    testb $32, %dl
-; X86-BMI2-NEXT:    je .LBB37_2
+; X86-BMI2-NEXT:    je .LBB38_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %edi, %esi
 ; X86-BMI2-NEXT:    xorl %edi, %edi
-; X86-BMI2-NEXT:  .LBB37_2:
+; X86-BMI2-NEXT:  .LBB38_2:
 ; X86-BMI2-NEXT:    movl 4(%eax), %edx
 ; X86-BMI2-NEXT:    andl %edi, %edx
 ; X86-BMI2-NEXT:    movl (%eax), %eax
@@ -2863,11 +2919,9 @@
 ; X64-BMI2-LABEL: bzhi64_c3_load_indexzext:
 ; X64-BMI2:       # %bb.0:
 ; X64-BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-BMI2-NEXT:    movq $-1, %rax
+; X64-BMI2-NEXT:    bzhiq %rsi, %rax, %rcx
 ; X64-BMI2-NEXT:    bzhiq %rsi, (%rdi), %rax
-; X64-BMI2-NEXT:    # kill: def $sil killed $sil killed $rsi def $rsi
-; X64-BMI2-NEXT:    negb %sil
-; X64-BMI2-NEXT:    movq $-1, %rcx
-; X64-BMI2-NEXT:    shrxq %rsi, %rcx, %rcx
 ; X64-BMI2-NEXT:    movq %rcx, (%rdx)
 ; X64-BMI2-NEXT:    retq
   %val = load i64, ptr %w
@@ -2890,11 +2944,11 @@
 ; X86-NOBMI-NEXT:    movl $-1, %edx
 ; X86-NOBMI-NEXT:    shrl %cl, %edx
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB38_2
+; X86-NOBMI-NEXT:    je .LBB39_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %edx, %eax
 ; X86-NOBMI-NEXT:    xorl %edx, %edx
-; X86-NOBMI-NEXT:  .LBB38_2:
+; X86-NOBMI-NEXT:  .LBB39_2:
 ; X86-NOBMI-NEXT:    movl %edx, 4(%esi)
 ; X86-NOBMI-NEXT:    movl %eax, (%esi)
 ; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
@@ -2912,11 +2966,11 @@
 ; X86-BMI1-NEXT:    movl $-1, %edx
 ; X86-BMI1-NEXT:    shrl %cl, %edx
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    je .LBB38_2
+; X86-BMI1-NEXT:    je .LBB39_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %edx, %eax
 ; X86-BMI1-NEXT:    xorl %edx, %edx
-; X86-BMI1-NEXT:  .LBB38_2:
+; X86-BMI1-NEXT:  .LBB39_2:
 ; X86-BMI1-NEXT:    movl %edx, 4(%esi)
 ; X86-BMI1-NEXT:    movl %eax, (%esi)
 ; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %eax
@@ -2933,11 +2987,11 @@
 ; X86-BMI2-NEXT:    movl $-1, %eax
 ; X86-BMI2-NEXT:    shrxl %ebx, %eax, %edx
 ; X86-BMI2-NEXT:    testb $32, %bl
-; X86-BMI2-NEXT:    je .LBB38_2
+; X86-BMI2-NEXT:    je .LBB39_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %edx, %eax
 ; X86-BMI2-NEXT:    xorl %edx, %edx
-; X86-BMI2-NEXT:  .LBB38_2:
+; X86-BMI2-NEXT:  .LBB39_2:
 ; X86-BMI2-NEXT:    movl %edx, 4(%ecx)
 ; X86-BMI2-NEXT:    movl %eax, (%ecx)
 ; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %eax
@@ -2969,12 +3023,10 @@
 ;
 ; X64-BMI2-LABEL: bzhi64_c4_commutative:
 ; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    movq $-1, %rax
+; X64-BMI2-NEXT:    bzhiq %rsi, %rax, %rax
+; X64-BMI2-NEXT:    movq %rax, (%rdx)
 ; X64-BMI2-NEXT:    bzhiq %rsi, %rdi, %rax
-; X64-BMI2-NEXT:    # kill: def $sil killed $sil killed $rsi def $rsi
-; X64-BMI2-NEXT:    negb %sil
-; X64-BMI2-NEXT:    movq $-1, %rcx
-; X64-BMI2-NEXT:    shrxq %rsi, %rcx, %rcx
-; X64-BMI2-NEXT:    movq %rcx, (%rdx)
 ; X64-BMI2-NEXT:    retq
   %numhighbits = sub i64 64, %numlowbits
   %mask = lshr i64 -1, %numhighbits
@@ -2994,10 +3046,10 @@
 ; X86-NOBMI-NEXT:    movl $-1, %eax
 ; X86-NOBMI-NEXT:    shrl %cl, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB39_2
+; X86-NOBMI-NEXT:    jne .LBB40_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl $-1, %eax
-; X86-NOBMI-NEXT:  .LBB39_2:
+; X86-NOBMI-NEXT:  .LBB40_2:
 ; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-NOBMI-NEXT:    retl
 ;
@@ -3008,10 +3060,10 @@
 ; X86-BMI1-NEXT:    movl $-1, %eax
 ; X86-BMI1-NEXT:    shrl %cl, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    jne .LBB39_2
+; X86-BMI1-NEXT:    jne .LBB40_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl $-1, %eax
-; X86-BMI1-NEXT:  .LBB39_2:
+; X86-BMI1-NEXT:  .LBB40_2:
 ; X86-BMI1-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-BMI1-NEXT:    retl
 ;
@@ -3021,10 +3073,10 @@
 ; X86-BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI2-NEXT:    movl $-1, %eax
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    je .LBB39_2
+; X86-BMI2-NEXT:    je .LBB40_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    shrxl %ecx, %eax, %eax
-; X86-BMI2-NEXT:  .LBB39_2:
+; X86-BMI2-NEXT:  .LBB40_2:
 ; X86-BMI2-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    retl
 ;
@@ -3175,10 +3227,10 @@
 ; X86-NEXT:    movl $-1, %edx
 ; X86-NEXT:    shrdl %cl, %eax, %edx
 ; X86-NEXT:    testb $32, %cl
-; X86-NEXT:    jne .LBB42_2
+; X86-NEXT:    jne .LBB43_2
 ; X86-NEXT:  # %bb.1:
 ; X86-NEXT:    movl %edx, %eax
-; X86-NEXT:  .LBB42_2:
+; X86-NEXT:  .LBB43_2:
 ; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    retl
 ;
@@ -3450,26 +3502,26 @@
 ; X86-NOBMI-NEXT:    shldl %cl, %edx, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl %esi, %edi
-; X86-NOBMI-NEXT:    jne .LBB47_2
+; X86-NOBMI-NEXT:    jne .LBB48_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %eax, %edi
-; X86-NOBMI-NEXT:  .LBB47_2:
+; X86-NOBMI-NEXT:  .LBB48_2:
 ; X86-NOBMI-NEXT:    movl %edi, %eax
 ; X86-NOBMI-NEXT:    shrl %cl, %eax
 ; X86-NOBMI-NEXT:    xorl %ebx, %ebx
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl $0, %edx
-; X86-NOBMI-NEXT:    jne .LBB47_4
+; X86-NOBMI-NEXT:    jne .LBB48_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %esi, %ebx
 ; X86-NOBMI-NEXT:    movl %eax, %edx
-; X86-NOBMI-NEXT:  .LBB47_4:
+; X86-NOBMI-NEXT:  .LBB48_4:
 ; X86-NOBMI-NEXT:    shrdl %cl, %edi, %ebx
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB47_6
+; X86-NOBMI-NEXT:    jne .LBB48_6
 ; X86-NOBMI-NEXT:  # %bb.5:
 ; X86-NOBMI-NEXT:    movl %ebx, %eax
-; X86-NOBMI-NEXT:  .LBB47_6:
+; X86-NOBMI-NEXT:  .LBB48_6:
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    popl %ebx
@@ -3489,26 +3541,26 @@
 ; X86-BMI1-NEXT:    shldl %cl, %edx, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
 ; X86-BMI1-NEXT:    movl %esi, %edi
-; X86-BMI1-NEXT:    jne .LBB47_2
+; X86-BMI1-NEXT:    jne .LBB48_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %eax, %edi
-; X86-BMI1-NEXT:  .LBB47_2:
+; X86-BMI1-NEXT:  .LBB48_2:
 ; X86-BMI1-NEXT:    movl %edi, %eax
 ; X86-BMI1-NEXT:    shrl %cl, %eax
 ; X86-BMI1-NEXT:    xorl %ebx, %ebx
 ; X86-BMI1-NEXT:    testb $32, %cl
 ; X86-BMI1-NEXT:    movl $0, %edx
-; X86-BMI1-NEXT:    jne .LBB47_4
+; X86-BMI1-NEXT:    jne .LBB48_4
 ; X86-BMI1-NEXT:  # %bb.3:
 ; X86-BMI1-NEXT:    movl %esi, %ebx
 ; X86-BMI1-NEXT:    movl %eax, %edx
-; X86-BMI1-NEXT:  .LBB47_4:
+; X86-BMI1-NEXT:  .LBB48_4:
 ; X86-BMI1-NEXT:    shrdl %cl, %edi, %ebx
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    jne .LBB47_6
+; X86-BMI1-NEXT:    jne .LBB48_6
 ; X86-BMI1-NEXT:  # %bb.5:
 ; X86-BMI1-NEXT:    movl %ebx, %eax
-; X86-BMI1-NEXT:  .LBB47_6:
+; X86-BMI1-NEXT:  .LBB48_6:
 ; X86-BMI1-NEXT:    popl %esi
 ; X86-BMI1-NEXT:    popl %edi
 ; X86-BMI1-NEXT:    popl %ebx
@@ -3526,22 +3578,22 @@
 ; X86-BMI2-NEXT:    shlxl %ecx, %eax, %edi
 ; X86-BMI2-NEXT:    xorl %edx, %edx
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    je .LBB47_2
+; X86-BMI2-NEXT:    je .LBB48_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %edi, %esi
 ; X86-BMI2-NEXT:    movl $0, %edi
-; X86-BMI2-NEXT:  .LBB47_2:
+; X86-BMI2-NEXT:  .LBB48_2:
 ; X86-BMI2-NEXT:    shrxl %ecx, %esi, %eax
-; X86-BMI2-NEXT:    jne .LBB47_4
+; X86-BMI2-NEXT:    jne .LBB48_4
 ; X86-BMI2-NEXT:  # %bb.3:
 ; X86-BMI2-NEXT:    movl %eax, %edx
-; X86-BMI2-NEXT:  .LBB47_4:
+; X86-BMI2-NEXT:  .LBB48_4:
 ; X86-BMI2-NEXT:    shrdl %cl, %esi, %edi
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    jne .LBB47_6
+; X86-BMI2-NEXT:    jne .LBB48_6
 ; X86-BMI2-NEXT:  # %bb.5:
 ; X86-BMI2-NEXT:    movl %edi, %eax
-; X86-BMI2-NEXT:  .LBB47_6:
+; X86-BMI2-NEXT:  .LBB48_6:
 ; X86-BMI2-NEXT:    popl %esi
 ; X86-BMI2-NEXT:    popl %edi
 ; X86-BMI2-NEXT:    retl
@@ -3587,26 +3639,26 @@
 ; X86-NOBMI-NEXT:    shldl %cl, %edx, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl %esi, %edi
-; X86-NOBMI-NEXT:    jne .LBB48_2
+; X86-NOBMI-NEXT:    jne .LBB49_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %eax, %edi
-; X86-NOBMI-NEXT:  .LBB48_2:
+; X86-NOBMI-NEXT:  .LBB49_2:
 ; X86-NOBMI-NEXT:    movl %edi, %eax
 ; X86-NOBMI-NEXT:    shrl %cl, %eax
 ; X86-NOBMI-NEXT:    xorl %ebx, %ebx
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl $0, %edx
-; X86-NOBMI-NEXT:    jne .LBB48_4
+; X86-NOBMI-NEXT:    jne .LBB49_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %esi, %ebx
 ; X86-NOBMI-NEXT:    movl %eax, %edx
-; X86-NOBMI-NEXT:  .LBB48_4:
+; X86-NOBMI-NEXT:  .LBB49_4:
 ; X86-NOBMI-NEXT:    shrdl %cl, %edi, %ebx
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB48_6
+; X86-NOBMI-NEXT:    jne .LBB49_6
 ; X86-NOBMI-NEXT:  # %bb.5:
 ; X86-NOBMI-NEXT:    movl %ebx, %eax
-; X86-NOBMI-NEXT:  .LBB48_6:
+; X86-NOBMI-NEXT:  .LBB49_6:
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    popl %ebx
@@ -3626,26 +3678,26 @@
 ; X86-BMI1-NEXT:    shldl %cl, %edx, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
 ; X86-BMI1-NEXT:    movl %esi, %edi
-; X86-BMI1-NEXT:    jne .LBB48_2
+; X86-BMI1-NEXT:    jne .LBB49_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %eax, %edi
-; X86-BMI1-NEXT:  .LBB48_2:
+; X86-BMI1-NEXT:  .LBB49_2:
 ; X86-BMI1-NEXT:    movl %edi, %eax
 ; X86-BMI1-NEXT:    shrl %cl, %eax
 ; X86-BMI1-NEXT:    xorl %ebx, %ebx
 ; X86-BMI1-NEXT:    testb $32, %cl
 ; X86-BMI1-NEXT:    movl $0, %edx
-; X86-BMI1-NEXT:    jne .LBB48_4
+; X86-BMI1-NEXT:    jne .LBB49_4
 ; X86-BMI1-NEXT:  # %bb.3:
 ; X86-BMI1-NEXT:    movl %esi, %ebx
 ; X86-BMI1-NEXT:    movl %eax, %edx
-; X86-BMI1-NEXT:  .LBB48_4:
+; X86-BMI1-NEXT:  .LBB49_4:
 ; X86-BMI1-NEXT:    shrdl %cl, %edi, %ebx
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    jne .LBB48_6
+; X86-BMI1-NEXT:    jne .LBB49_6
 ; X86-BMI1-NEXT:  # %bb.5:
 ; X86-BMI1-NEXT:    movl %ebx, %eax
-; X86-BMI1-NEXT:  .LBB48_6:
+; X86-BMI1-NEXT:  .LBB49_6:
 ; X86-BMI1-NEXT:    popl %esi
 ; X86-BMI1-NEXT:    popl %edi
 ; X86-BMI1-NEXT:    popl %ebx
@@ -3663,22 +3715,22 @@
 ; X86-BMI2-NEXT:    shlxl %ecx, %eax, %edi
 ; X86-BMI2-NEXT:    xorl %edx, %edx
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    je .LBB48_2
+; X86-BMI2-NEXT:    je .LBB49_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %edi, %esi
 ; X86-BMI2-NEXT:    movl $0, %edi
-; X86-BMI2-NEXT:  .LBB48_2:
+; X86-BMI2-NEXT:  .LBB49_2:
 ; X86-BMI2-NEXT:    shrxl %ecx, %esi, %eax
-; X86-BMI2-NEXT:    jne .LBB48_4
+; X86-BMI2-NEXT:    jne .LBB49_4
 ; X86-BMI2-NEXT:  # %bb.3:
 ; X86-BMI2-NEXT:    movl %eax, %edx
-; X86-BMI2-NEXT:  .LBB48_4:
+; X86-BMI2-NEXT:  .LBB49_4:
 ; X86-BMI2-NEXT:    shrdl %cl, %esi, %edi
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    jne .LBB48_6
+; X86-BMI2-NEXT:    jne .LBB49_6
 ; X86-BMI2-NEXT:  # %bb.5:
 ; X86-BMI2-NEXT:    movl %edi, %eax
-; X86-BMI2-NEXT:  .LBB48_6:
+; X86-BMI2-NEXT:  .LBB49_6:
 ; X86-BMI2-NEXT:    popl %esi
 ; X86-BMI2-NEXT:    popl %edi
 ; X86-BMI2-NEXT:    retl
@@ -3728,26 +3780,26 @@
 ; X86-NOBMI-NEXT:    shldl %cl, %edx, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl %esi, %edi
-; X86-NOBMI-NEXT:    jne .LBB49_2
+; X86-NOBMI-NEXT:    jne .LBB50_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %eax, %edi
-; X86-NOBMI-NEXT:  .LBB49_2:
+; X86-NOBMI-NEXT:  .LBB50_2:
 ; X86-NOBMI-NEXT:    movl %edi, %eax
 ; X86-NOBMI-NEXT:    shrl %cl, %eax
 ; X86-NOBMI-NEXT:    xorl %ebx, %ebx
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl $0, %edx
-; X86-NOBMI-NEXT:    jne .LBB49_4
+; X86-NOBMI-NEXT:    jne .LBB50_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %esi, %ebx
 ; X86-NOBMI-NEXT:    movl %eax, %edx
-; X86-NOBMI-NEXT:  .LBB49_4:
+; X86-NOBMI-NEXT:  .LBB50_4:
 ; X86-NOBMI-NEXT:    shrdl %cl, %edi, %ebx
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB49_6
+; X86-NOBMI-NEXT:    jne .LBB50_6
 ; X86-NOBMI-NEXT:  # %bb.5:
 ; X86-NOBMI-NEXT:    movl %ebx, %eax
-; X86-NOBMI-NEXT:  .LBB49_6:
+; X86-NOBMI-NEXT:  .LBB50_6:
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    popl %ebx
@@ -3768,26 +3820,26 @@
 ; X86-BMI1-NEXT:    shldl %cl, %edx, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
 ; X86-BMI1-NEXT:    movl %esi, %edi
-; X86-BMI1-NEXT:    jne .LBB49_2
+; X86-BMI1-NEXT:    jne .LBB50_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %eax, %edi
-; X86-BMI1-NEXT:  .LBB49_2:
+; X86-BMI1-NEXT:  .LBB50_2:
 ; X86-BMI1-NEXT:    movl %edi, %eax
 ; X86-BMI1-NEXT:    shrl %cl, %eax
 ; X86-BMI1-NEXT:    xorl %ebx, %ebx
 ; X86-BMI1-NEXT:    testb $32, %cl
 ; X86-BMI1-NEXT:    movl $0, %edx
-; X86-BMI1-NEXT:    jne .LBB49_4
+; X86-BMI1-NEXT:    jne .LBB50_4
 ; X86-BMI1-NEXT:  # %bb.3:
 ; X86-BMI1-NEXT:    movl %esi, %ebx
 ; X86-BMI1-NEXT:    movl %eax, %edx
-; X86-BMI1-NEXT:  .LBB49_4:
+; X86-BMI1-NEXT:  .LBB50_4:
 ; X86-BMI1-NEXT:    shrdl %cl, %edi, %ebx
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    jne .LBB49_6
+; X86-BMI1-NEXT:    jne .LBB50_6
 ; X86-BMI1-NEXT:  # %bb.5:
 ; X86-BMI1-NEXT:    movl %ebx, %eax
-; X86-BMI1-NEXT:  .LBB49_6:
+; X86-BMI1-NEXT:  .LBB50_6:
 ; X86-BMI1-NEXT:    popl %esi
 ; X86-BMI1-NEXT:    popl %edi
 ; X86-BMI1-NEXT:    popl %ebx
@@ -3806,22 +3858,22 @@
 ; X86-BMI2-NEXT:    shlxl %ecx, %edx, %edi
 ; X86-BMI2-NEXT:    xorl %edx, %edx
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    je .LBB49_2
+; X86-BMI2-NEXT:    je .LBB50_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %edi, %esi
 ; X86-BMI2-NEXT:    movl $0, %edi
-; X86-BMI2-NEXT:  .LBB49_2:
+; X86-BMI2-NEXT:  .LBB50_2:
 ; X86-BMI2-NEXT:    shrxl %ecx, %esi, %eax
-; X86-BMI2-NEXT:    jne .LBB49_4
+; X86-BMI2-NEXT:    jne .LBB50_4
 ; X86-BMI2-NEXT:  # %bb.3:
 ; X86-BMI2-NEXT:    movl %eax, %edx
-; X86-BMI2-NEXT:  .LBB49_4:
+; X86-BMI2-NEXT:  .LBB50_4:
 ; X86-BMI2-NEXT:    shrdl %cl, %esi, %edi
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    jne .LBB49_6
+; X86-BMI2-NEXT:    jne .LBB50_6
 ; X86-BMI2-NEXT:  # %bb.5:
 ; X86-BMI2-NEXT:    movl %edi, %eax
-; X86-BMI2-NEXT:  .LBB49_6:
+; X86-BMI2-NEXT:  .LBB50_6:
 ; X86-BMI2-NEXT:    popl %esi
 ; X86-BMI2-NEXT:    popl %edi
 ; X86-BMI2-NEXT:    retl
@@ -3869,26 +3921,26 @@
 ; X86-NOBMI-NEXT:    shldl %cl, %edx, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl %esi, %edi
-; X86-NOBMI-NEXT:    jne .LBB50_2
+; X86-NOBMI-NEXT:    jne .LBB51_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %eax, %edi
-; X86-NOBMI-NEXT:  .LBB50_2:
+; X86-NOBMI-NEXT:  .LBB51_2:
 ; X86-NOBMI-NEXT:    movl %edi, %eax
 ; X86-NOBMI-NEXT:    shrl %cl, %eax
 ; X86-NOBMI-NEXT:    xorl %ebx, %ebx
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl $0, %edx
-; X86-NOBMI-NEXT:    jne .LBB50_4
+; X86-NOBMI-NEXT:    jne .LBB51_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %esi, %ebx
 ; X86-NOBMI-NEXT:    movl %eax, %edx
-; X86-NOBMI-NEXT:  .LBB50_4:
+; X86-NOBMI-NEXT:  .LBB51_4:
 ; X86-NOBMI-NEXT:    shrdl %cl, %edi, %ebx
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB50_6
+; X86-NOBMI-NEXT:    jne .LBB51_6
 ; X86-NOBMI-NEXT:  # %bb.5:
 ; X86-NOBMI-NEXT:    movl %ebx, %eax
-; X86-NOBMI-NEXT:  .LBB50_6:
+; X86-NOBMI-NEXT:  .LBB51_6:
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    popl %ebx
@@ -3909,26 +3961,26 @@
 ; X86-BMI1-NEXT:    shldl %cl, %edx, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
 ; X86-BMI1-NEXT:    movl %esi, %edi
-; X86-BMI1-NEXT:    jne .LBB50_2
+; X86-BMI1-NEXT:    jne .LBB51_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %eax, %edi
-; X86-BMI1-NEXT:  .LBB50_2:
+; X86-BMI1-NEXT:  .LBB51_2:
 ; X86-BMI1-NEXT:    movl %edi, %eax
 ; X86-BMI1-NEXT:    shrl %cl, %eax
 ; X86-BMI1-NEXT:    xorl %ebx, %ebx
 ; X86-BMI1-NEXT:    testb $32, %cl
 ; X86-BMI1-NEXT:    movl $0, %edx
-; X86-BMI1-NEXT:    jne .LBB50_4
+; X86-BMI1-NEXT:    jne .LBB51_4
 ; X86-BMI1-NEXT:  # %bb.3:
 ; X86-BMI1-NEXT:    movl %esi, %ebx
 ; X86-BMI1-NEXT:    movl %eax, %edx
-; X86-BMI1-NEXT:  .LBB50_4:
+; X86-BMI1-NEXT:  .LBB51_4:
 ; X86-BMI1-NEXT:    shrdl %cl, %edi, %ebx
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    jne .LBB50_6
+; X86-BMI1-NEXT:    jne .LBB51_6
 ; X86-BMI1-NEXT:  # %bb.5:
 ; X86-BMI1-NEXT:    movl %ebx, %eax
-; X86-BMI1-NEXT:  .LBB50_6:
+; X86-BMI1-NEXT:  .LBB51_6:
 ; X86-BMI1-NEXT:    popl %esi
 ; X86-BMI1-NEXT:    popl %edi
 ; X86-BMI1-NEXT:    popl %ebx
@@ -3947,22 +3999,22 @@
 ; X86-BMI2-NEXT:    shlxl %ecx, %edx, %edi
 ; X86-BMI2-NEXT:    xorl %edx, %edx
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    je .LBB50_2
+; X86-BMI2-NEXT:    je .LBB51_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %edi, %esi
 ; X86-BMI2-NEXT:    movl $0, %edi
-; X86-BMI2-NEXT:  .LBB50_2:
+; X86-BMI2-NEXT:  .LBB51_2:
 ; X86-BMI2-NEXT:    shrxl %ecx, %esi, %eax
-; X86-BMI2-NEXT:    jne .LBB50_4
+; X86-BMI2-NEXT:    jne .LBB51_4
 ; X86-BMI2-NEXT:  # %bb.3:
 ; X86-BMI2-NEXT:    movl %eax, %edx
-; X86-BMI2-NEXT:  .LBB50_4:
+; X86-BMI2-NEXT:  .LBB51_4:
 ; X86-BMI2-NEXT:    shrdl %cl, %esi, %edi
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    jne .LBB50_6
+; X86-BMI2-NEXT:    jne .LBB51_6
 ; X86-BMI2-NEXT:  # %bb.5:
 ; X86-BMI2-NEXT:    movl %edi, %eax
-; X86-BMI2-NEXT:  .LBB50_6:
+; X86-BMI2-NEXT:  .LBB51_6:
 ; X86-BMI2-NEXT:    popl %esi
 ; X86-BMI2-NEXT:    popl %edi
 ; X86-BMI2-NEXT:    retl
@@ -4012,18 +4064,18 @@
 ; X86-NOBMI-NEXT:    shll %cl, %edx
 ; X86-NOBMI-NEXT:    shldl %cl, %esi, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB51_2
+; X86-NOBMI-NEXT:    je .LBB52_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %edx, %eax
 ; X86-NOBMI-NEXT:    xorl %edx, %edx
-; X86-NOBMI-NEXT:  .LBB51_2:
+; X86-NOBMI-NEXT:  .LBB52_2:
 ; X86-NOBMI-NEXT:    shrdl %cl, %eax, %edx
 ; X86-NOBMI-NEXT:    shrl %cl, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB51_4
+; X86-NOBMI-NEXT:    jne .LBB52_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %edx, %eax
-; X86-NOBMI-NEXT:  .LBB51_4:
+; X86-NOBMI-NEXT:  .LBB52_4:
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    retl
 ;
@@ -4038,18 +4090,18 @@
 ; X86-BMI1-NEXT:    shll %cl, %edx
 ; X86-BMI1-NEXT:    shldl %cl, %esi, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    je .LBB51_2
+; X86-BMI1-NEXT:    je .LBB52_2
 ; X86-BMI1-NEXT:  # %bb.1:
 ; X86-BMI1-NEXT:    movl %edx, %eax
 ; X86-BMI1-NEXT:    xorl %edx, %edx
-; X86-BMI1-NEXT:  .LBB51_2:
+; X86-BMI1-NEXT:  .LBB52_2:
 ; X86-BMI1-NEXT:    shrdl %cl, %eax, %edx
 ; X86-BMI1-NEXT:    shrl %cl, %eax
 ; X86-BMI1-NEXT:    testb $32, %cl
-; X86-BMI1-NEXT:    jne .LBB51_4
+; X86-BMI1-NEXT:    jne .LBB52_4
 ; X86-BMI1-NEXT:  # %bb.3:
 ; X86-BMI1-NEXT:    movl %edx, %eax
-; X86-BMI1-NEXT:  .LBB51_4:
+; X86-BMI1-NEXT:  .LBB52_4:
 ; X86-BMI1-NEXT:    popl %esi
 ; X86-BMI1-NEXT:    retl
 ;
@@ -4062,17 +4114,17 @@
 ; X86-BMI2-NEXT:    shldl %cl, %eax, %edx
 ; X86-BMI2-NEXT:    shlxl %ecx, %eax, %eax
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    je .LBB51_2
+; X86-BMI2-NEXT:    je .LBB52_2
 ; X86-BMI2-NEXT:  # %bb.1:
 ; X86-BMI2-NEXT:    movl %eax, %edx
 ; X86-BMI2-NEXT:    xorl %eax, %eax
-; X86-BMI2-NEXT:  .LBB51_2:
+; X86-BMI2-NEXT:  .LBB52_2:
 ; X86-BMI2-NEXT:    shrdl %cl, %edx, %eax
 ; X86-BMI2-NEXT:    testb $32, %cl
-; X86-BMI2-NEXT:    je .LBB51_4
+; X86-BMI2-NEXT:    je .LBB52_4
 ; X86-BMI2-NEXT:  # %bb.3:
 ; X86-BMI2-NEXT:    shrxl %ecx, %edx, %eax
-; X86-BMI2-NEXT:  .LBB51_4:
+; X86-BMI2-NEXT:  .LBB52_4:
 ; X86-BMI2-NEXT:    retl
 ;
 ; X64-NOBMI-LABEL: bzhi64_32_d0: