[TargetLowering] Expand the last stage of i16 popcnt using shift+add+and instead of mul+shift.

[craig.topper]

Summary

If we use multiply it would be with 0x0101 which is 1 more than a power
of 2. On some targets we would expand this to shl+add. By avoiding the
multiply earlier, we can generate better code.

Note, PowerPC doesn't do the shl+add expansion of multiply so one of
the tests increased in instruction count.

Limiting to scalars because it almost always increased the number of
instructions in vector tests.

[RKSimon]

LGTM

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

Op = DAG.getNode(ISD::AND, dl, VT,
                    DAG.getNode(ISD::ADD, dl, VT, Op,
                                DAG.getNode(ISD::SRL, dl, VT, Op,
                                            DAG.getConstant(4, dl, ShVT))),
                    Mask0F);
 
   if (Len <= 8)
     return Op;
 
+  // Avoid the multiply if we only have 2 bytes to add.
+  // TODO: Only doing this for scalars because vectors weren't as obviously
+  // improved.
+  if (Len == 16 && !VT.isVector()) {
+    // v = (v + (v >> 8)) & 0x00FF;
+    return DAG.getNode(ISD::AND, dl, VT,
+                     DAG.getNode(ISD::ADD, dl, VT, Op,
+                                 DAG.getNode(ISD::SRL, dl, VT, Op,
+                                             DAG.getConstant(8, dl, ShVT))),
+                     DAG.getConstant(0xFF, dl, VT));
+  }
+
   // v = (v * 0x01010101...) >> (Len - 8)
   SDValue Mask01 =
       DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT);
   return DAG.getNode(ISD::SRL, dl, VT,
                      DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
                      DAG.getConstant(Len - 8, dl, ShVT));
 }
 

llvm/test/CodeGen/PowerPC/popcnt-zext.ll

 ; SLOW-NEXT:    andi. 3, 3, 85
 ; SLOW-NEXT:    sub 3, 4, 3
 ; SLOW-NEXT:    andi. 4, 3, 13107
 ; SLOW-NEXT:    rotlwi 3, 3, 30
 ; SLOW-NEXT:    andi. 3, 3, 13107
 ; SLOW-NEXT:    add 3, 4, 3
 ; SLOW-NEXT:    srwi 4, 3, 4
 ; SLOW-NEXT:    add 3, 3, 4
-; SLOW-NEXT:    andi. 3, 3, 3855
-; SLOW-NEXT:    mulli 3, 3, 257
-; SLOW-NEXT:    rlwinm 3, 3, 24, 24, 31
+; SLOW-NEXT:    rlwinm 4, 3, 24, 28, 31
+; SLOW-NEXT:    clrlwi 3, 3, 28
+; SLOW-NEXT:    add 3, 3, 4
 ; SLOW-NEXT:    blr
   %z = zext i8 %x to i16
   %pop = tail call i16 @llvm.ctpop.i16(i16 %z)
   ret i16 %pop
 }
 
 define i16 @popz_i8_i16(i8 %x) {
 ; FAST-LABEL: popz_i8_i16:

 ; SLOW-NEXT:    andi. 4, 4, 21845
 ; SLOW-NEXT:    sub 3, 3, 4
 ; SLOW-NEXT:    andi. 4, 3, 13107
 ; SLOW-NEXT:    rotlwi 3, 3, 30
 ; SLOW-NEXT:    andi. 3, 3, 13107
 ; SLOW-NEXT:    add 3, 4, 3
 ; SLOW-NEXT:    srwi 4, 3, 4
 ; SLOW-NEXT:    add 3, 3, 4
-; SLOW-NEXT:    andi. 3, 3, 3855
-; SLOW-NEXT:    mulli 3, 3, 257
-; SLOW-NEXT:    rlwinm 3, 3, 24, 24, 31
+; SLOW-NEXT:    rlwinm 4, 3, 24, 28, 31
+; SLOW-NEXT:    clrlwi 3, 3, 28
+; SLOW-NEXT:    add 3, 3, 4
+; SLOW-NEXT:    clrldi 3, 3, 32
 ; SLOW-NEXT:    blr
   %pop = tail call i16 @llvm.ctpop.i16(i16 %x)
   %z = zext i16 %pop to i32
   ret i32 %z
 }
 
 define i64 @zpop_i32_i64(i32 %x) {
 ; FAST-LABEL: zpop_i32_i64:

 ; SLOW-NEXT:    andi. 4, 4, 21845
 ; SLOW-NEXT:    sub 3, 3, 4
 ; SLOW-NEXT:    andi. 4, 3, 13107
 ; SLOW-NEXT:    rotlwi 3, 3, 30
 ; SLOW-NEXT:    andi. 3, 3, 13107
 ; SLOW-NEXT:    add 3, 4, 3
 ; SLOW-NEXT:    srwi 4, 3, 4
 ; SLOW-NEXT:    add 3, 3, 4
-; SLOW-NEXT:    andi. 3, 3, 3855
-; SLOW-NEXT:    mulli 3, 3, 257
-; SLOW-NEXT:    srwi 3, 3, 8
+; SLOW-NEXT:    rlwinm 4, 3, 24, 28, 31
+; SLOW-NEXT:    clrlwi 3, 3, 28
+; SLOW-NEXT:    add 3, 3, 4
 ; SLOW-NEXT:    rlwinm 3, 3, 0, 27, 27
 ; SLOW-NEXT:    blr
   %pop = call i16 @llvm.ctpop.i16(i16 %x)
   %z = zext i16 %pop to i64 ; SimplifyDemandedBits may turn zext (or sext) into aext
   %a = and i64 %z, 16
   ret i64 %a
 }
 
 declare i8 @llvm.ctpop.i8(i8) nounwind readnone
 declare i16 @llvm.ctpop.i16(i16) nounwind readnone
 declare i32 @llvm.ctpop.i32(i32) nounwind readnone
 declare i64 @llvm.ctpop.i64(i64) nounwind readnone

llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll

 ; RV32_NOZBB-NEXT:    lui a1, 3
 ; RV32_NOZBB-NEXT:    addi a1, a1, 819
 ; RV32_NOZBB-NEXT:    and a2, a0, a1
 ; RV32_NOZBB-NEXT:    srli a0, a0, 2
 ; RV32_NOZBB-NEXT:    and a0, a0, a1
 ; RV32_NOZBB-NEXT:    add a0, a2, a0
 ; RV32_NOZBB-NEXT:    srli a1, a0, 4
 ; RV32_NOZBB-NEXT:    add a0, a0, a1
-; RV32_NOZBB-NEXT:    lui a1, 1
-; RV32_NOZBB-NEXT:    addi a1, a1, -241
-; RV32_NOZBB-NEXT:    and a0, a0, a1
-; RV32_NOZBB-NEXT:    slli a1, a0, 8
+; RV32_NOZBB-NEXT:    andi a1, a0, 15
+; RV32_NOZBB-NEXT:    slli a0, a0, 20
+; RV32_NOZBB-NEXT:    srli a0, a0, 28
 ; RV32_NOZBB-NEXT:    add a0, a1, a0
-; RV32_NOZBB-NEXT:    slli a0, a0, 19
-; RV32_NOZBB-NEXT:    srli a0, a0, 27
 ; RV32_NOZBB-NEXT:    ret
 ; RV32_NOZBB-NEXT:  .LBB1_2:
 ; RV32_NOZBB-NEXT:    li a0, 16
 ; RV32_NOZBB-NEXT:    ret
 ;
 ; RV64NOZBB-LABEL: test_cttz_i16:
 ; RV64NOZBB:       # %bb.0:
 ; RV64NOZBB-NEXT:    slli a1, a0, 48

 ; RV64NOZBB-NEXT:    sub a0, a0, a1
 ; RV64NOZBB-NEXT:    lui a1, 3
 ; RV64NOZBB-NEXT:    addiw a1, a1, 819
 ; RV64NOZBB-NEXT:    and a2, a0, a1
 ; RV64NOZBB-NEXT:    srli a0, a0, 2
 ; RV64NOZBB-NEXT:    and a0, a0, a1
 ; RV64NOZBB-NEXT:    add a0, a2, a0
 ; RV64NOZBB-NEXT:    srli a1, a0, 4
-; RV64NOZBB-NEXT:    add a0, a0, a1
-; RV64NOZBB-NEXT:    lui a1, 1
-; RV64NOZBB-NEXT:    addiw a1, a1, -241
-; RV64NOZBB-NEXT:    and a0, a0, a1
-; RV64NOZBB-NEXT:    slliw a1, a0, 8
-; RV64NOZBB-NEXT:    addw a0, a1, a0
-; RV64NOZBB-NEXT:    slli a0, a0, 51
-; RV64NOZBB-NEXT:    srli a0, a0, 59
+; RV64NOZBB-NEXT:    addw a0, a0, a1
+; RV64NOZBB-NEXT:    andi a1, a0, 15
+; RV64NOZBB-NEXT:    slli a0, a0, 52
+; RV64NOZBB-NEXT:    srli a0, a0, 60
+; RV64NOZBB-NEXT:    add a0, a1, a0
 ; RV64NOZBB-NEXT:    ret
 ; RV64NOZBB-NEXT:  .LBB1_2:
 ; RV64NOZBB-NEXT:    li a0, 16
 ; RV64NOZBB-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: test_cttz_i16:
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    lui a1, 16

 ; RV32_NOZBB-NEXT:    lui a1, 3
 ; RV32_NOZBB-NEXT:    addi a1, a1, 819
 ; RV32_NOZBB-NEXT:    and a2, a0, a1
 ; RV32_NOZBB-NEXT:    srli a0, a0, 2
 ; RV32_NOZBB-NEXT:    and a0, a0, a1
 ; RV32_NOZBB-NEXT:    add a0, a2, a0
 ; RV32_NOZBB-NEXT:    srli a1, a0, 4
 ; RV32_NOZBB-NEXT:    add a0, a0, a1
-; RV32_NOZBB-NEXT:    lui a1, 1
-; RV32_NOZBB-NEXT:    addi a1, a1, -241
-; RV32_NOZBB-NEXT:    and a0, a0, a1
-; RV32_NOZBB-NEXT:    slli a1, a0, 8
+; RV32_NOZBB-NEXT:    andi a1, a0, 15
+; RV32_NOZBB-NEXT:    slli a0, a0, 20
+; RV32_NOZBB-NEXT:    srli a0, a0, 28
 ; RV32_NOZBB-NEXT:    add a0, a1, a0
-; RV32_NOZBB-NEXT:    slli a0, a0, 19
-; RV32_NOZBB-NEXT:    srli a0, a0, 27
 ; RV32_NOZBB-NEXT:    ret
 ;
 ; RV64NOZBB-LABEL: test_cttz_i16_zero_undef:
 ; RV64NOZBB:       # %bb.0:
 ; RV64NOZBB-NEXT:    addi a1, a0, -1
 ; RV64NOZBB-NEXT:    not a0, a0
 ; RV64NOZBB-NEXT:    and a0, a0, a1
 ; RV64NOZBB-NEXT:    srli a1, a0, 1
 ; RV64NOZBB-NEXT:    lui a2, 5
 ; RV64NOZBB-NEXT:    addiw a2, a2, 1365
 ; RV64NOZBB-NEXT:    and a1, a1, a2
 ; RV64NOZBB-NEXT:    sub a0, a0, a1
 ; RV64NOZBB-NEXT:    lui a1, 3
 ; RV64NOZBB-NEXT:    addiw a1, a1, 819
 ; RV64NOZBB-NEXT:    and a2, a0, a1
 ; RV64NOZBB-NEXT:    srli a0, a0, 2
 ; RV64NOZBB-NEXT:    and a0, a0, a1
 ; RV64NOZBB-NEXT:    add a0, a2, a0
 ; RV64NOZBB-NEXT:    srli a1, a0, 4
-; RV64NOZBB-NEXT:    add a0, a0, a1
-; RV64NOZBB-NEXT:    lui a1, 1
-; RV64NOZBB-NEXT:    addiw a1, a1, -241
-; RV64NOZBB-NEXT:    and a0, a0, a1
-; RV64NOZBB-NEXT:    slliw a1, a0, 8
-; RV64NOZBB-NEXT:    addw a0, a1, a0
-; RV64NOZBB-NEXT:    slli a0, a0, 51
-; RV64NOZBB-NEXT:    srli a0, a0, 59
+; RV64NOZBB-NEXT:    addw a0, a0, a1
+; RV64NOZBB-NEXT:    andi a1, a0, 15
+; RV64NOZBB-NEXT:    slli a0, a0, 52
+; RV64NOZBB-NEXT:    srli a0, a0, 60
+; RV64NOZBB-NEXT:    add a0, a1, a0
 ; RV64NOZBB-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: test_cttz_i16_zero_undef:
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    ctz a0, a0
 ; RV32ZBB-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: test_cttz_i16_zero_undef:

 ; RV32_NOZBB-NEXT:    lui a1, 3
 ; RV32_NOZBB-NEXT:    addi a1, a1, 819
 ; RV32_NOZBB-NEXT:    and a2, a0, a1
 ; RV32_NOZBB-NEXT:    srli a0, a0, 2
 ; RV32_NOZBB-NEXT:    and a0, a0, a1
 ; RV32_NOZBB-NEXT:    add a0, a2, a0
 ; RV32_NOZBB-NEXT:    srli a1, a0, 4
 ; RV32_NOZBB-NEXT:    add a0, a0, a1
-; RV32_NOZBB-NEXT:    lui a1, 1
-; RV32_NOZBB-NEXT:    addi a1, a1, -241
-; RV32_NOZBB-NEXT:    and a0, a0, a1
-; RV32_NOZBB-NEXT:    slli a1, a0, 8
+; RV32_NOZBB-NEXT:    andi a1, a0, 15
+; RV32_NOZBB-NEXT:    slli a0, a0, 20
+; RV32_NOZBB-NEXT:    srli a0, a0, 28
 ; RV32_NOZBB-NEXT:    add a0, a1, a0
-; RV32_NOZBB-NEXT:    slli a0, a0, 19
-; RV32_NOZBB-NEXT:    srli a0, a0, 27
 ; RV32_NOZBB-NEXT:    ret
 ; RV32_NOZBB-NEXT:  .LBB9_2:
 ; RV32_NOZBB-NEXT:    li a0, 16
 ; RV32_NOZBB-NEXT:    ret
 ;
 ; RV64NOZBB-LABEL: test_ctlz_i16:
 ; RV64NOZBB:       # %bb.0:
 ; RV64NOZBB-NEXT:    slli a1, a0, 48

 ; RV64NOZBB-NEXT:    sub a0, a0, a1
 ; RV64NOZBB-NEXT:    lui a1, 3
 ; RV64NOZBB-NEXT:    addiw a1, a1, 819
 ; RV64NOZBB-NEXT:    and a2, a0, a1
 ; RV64NOZBB-NEXT:    srli a0, a0, 2
 ; RV64NOZBB-NEXT:    and a0, a0, a1
 ; RV64NOZBB-NEXT:    add a0, a2, a0
 ; RV64NOZBB-NEXT:    srli a1, a0, 4
-; RV64NOZBB-NEXT:    add a0, a0, a1
-; RV64NOZBB-NEXT:    lui a1, 1
-; RV64NOZBB-NEXT:    addiw a1, a1, -241
-; RV64NOZBB-NEXT:    and a0, a0, a1
-; RV64NOZBB-NEXT:    slliw a1, a0, 8
-; RV64NOZBB-NEXT:    addw a0, a1, a0
-; RV64NOZBB-NEXT:    slli a0, a0, 51
-; RV64NOZBB-NEXT:    srli a0, a0, 59
+; RV64NOZBB-NEXT:    addw a0, a0, a1
+; RV64NOZBB-NEXT:    andi a1, a0, 15
+; RV64NOZBB-NEXT:    slli a0, a0, 52
+; RV64NOZBB-NEXT:    srli a0, a0, 60
+; RV64NOZBB-NEXT:    add a0, a1, a0
 ; RV64NOZBB-NEXT:    ret
 ; RV64NOZBB-NEXT:  .LBB9_2:
 ; RV64NOZBB-NEXT:    li a0, 16
 ; RV64NOZBB-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: test_ctlz_i16:
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    zext.h a0, a0

 ; RV32_NOZBB-NEXT:    lui a1, 3
 ; RV32_NOZBB-NEXT:    addi a1, a1, 819
 ; RV32_NOZBB-NEXT:    and a2, a0, a1
 ; RV32_NOZBB-NEXT:    srli a0, a0, 2
 ; RV32_NOZBB-NEXT:    and a0, a0, a1
 ; RV32_NOZBB-NEXT:    add a0, a2, a0
 ; RV32_NOZBB-NEXT:    srli a1, a0, 4
 ; RV32_NOZBB-NEXT:    add a0, a0, a1
-; RV32_NOZBB-NEXT:    lui a1, 1
-; RV32_NOZBB-NEXT:    addi a1, a1, -241
-; RV32_NOZBB-NEXT:    and a0, a0, a1
-; RV32_NOZBB-NEXT:    slli a1, a0, 8
+; RV32_NOZBB-NEXT:    andi a1, a0, 15
+; RV32_NOZBB-NEXT:    slli a0, a0, 20
+; RV32_NOZBB-NEXT:    srli a0, a0, 28
 ; RV32_NOZBB-NEXT:    add a0, a1, a0
-; RV32_NOZBB-NEXT:    slli a0, a0, 19
-; RV32_NOZBB-NEXT:    srli a0, a0, 27
 ; RV32_NOZBB-NEXT:    ret
 ;
 ; RV64NOZBB-LABEL: test_ctlz_i16_zero_undef:
 ; RV64NOZBB:       # %bb.0:
 ; RV64NOZBB-NEXT:    slli a1, a0, 48
 ; RV64NOZBB-NEXT:    srli a1, a1, 49
 ; RV64NOZBB-NEXT:    or a0, a0, a1
 ; RV64NOZBB-NEXT:    slli a1, a0, 48

 ; RV64NOZBB-NEXT:    sub a0, a0, a1
 ; RV64NOZBB-NEXT:    lui a1, 3
 ; RV64NOZBB-NEXT:    addiw a1, a1, 819
 ; RV64NOZBB-NEXT:    and a2, a0, a1
 ; RV64NOZBB-NEXT:    srli a0, a0, 2
 ; RV64NOZBB-NEXT:    and a0, a0, a1
 ; RV64NOZBB-NEXT:    add a0, a2, a0
 ; RV64NOZBB-NEXT:    srli a1, a0, 4
-; RV64NOZBB-NEXT:    add a0, a0, a1
-; RV64NOZBB-NEXT:    lui a1, 1
-; RV64NOZBB-NEXT:    addiw a1, a1, -241
-; RV64NOZBB-NEXT:    and a0, a0, a1
-; RV64NOZBB-NEXT:    slliw a1, a0, 8
-; RV64NOZBB-NEXT:    addw a0, a1, a0
-; RV64NOZBB-NEXT:    slli a0, a0, 51
-; RV64NOZBB-NEXT:    srli a0, a0, 59
+; RV64NOZBB-NEXT:    addw a0, a0, a1
+; RV64NOZBB-NEXT:    andi a1, a0, 15
+; RV64NOZBB-NEXT:    slli a0, a0, 52
+; RV64NOZBB-NEXT:    srli a0, a0, 60
+; RV64NOZBB-NEXT:    add a0, a1, a0
 ; RV64NOZBB-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: test_ctlz_i16_zero_undef:
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    zext.h a0, a0
 ; RV32ZBB-NEXT:    clz a0, a0
 ; RV32ZBB-NEXT:    addi a0, a0, -16
 ; RV32ZBB-NEXT:    ret

 ; RV32_NOZBB-NEXT:    lui a1, 3
 ; RV32_NOZBB-NEXT:    addi a1, a1, 819
 ; RV32_NOZBB-NEXT:    and a2, a0, a1
 ; RV32_NOZBB-NEXT:    srli a0, a0, 2
 ; RV32_NOZBB-NEXT:    and a0, a0, a1
 ; RV32_NOZBB-NEXT:    add a0, a2, a0
 ; RV32_NOZBB-NEXT:    srli a1, a0, 4
 ; RV32_NOZBB-NEXT:    add a0, a0, a1
-; RV32_NOZBB-NEXT:    lui a1, 1
-; RV32_NOZBB-NEXT:    addi a1, a1, -241
-; RV32_NOZBB-NEXT:    and a0, a0, a1
-; RV32_NOZBB-NEXT:    slli a1, a0, 8
+; RV32_NOZBB-NEXT:    andi a1, a0, 15
+; RV32_NOZBB-NEXT:    slli a0, a0, 20
+; RV32_NOZBB-NEXT:    srli a0, a0, 28
 ; RV32_NOZBB-NEXT:    add a0, a1, a0
-; RV32_NOZBB-NEXT:    slli a0, a0, 19
-; RV32_NOZBB-NEXT:    srli a0, a0, 27
 ; RV32_NOZBB-NEXT:    ret
 ;
 ; RV64NOZBB-LABEL: test_ctpop_i16:
 ; RV64NOZBB:       # %bb.0:
 ; RV64NOZBB-NEXT:    srli a1, a0, 1
 ; RV64NOZBB-NEXT:    lui a2, 5
 ; RV64NOZBB-NEXT:    addiw a2, a2, 1365
 ; RV64NOZBB-NEXT:    and a1, a1, a2
 ; RV64NOZBB-NEXT:    sub a0, a0, a1
 ; RV64NOZBB-NEXT:    lui a1, 3
 ; RV64NOZBB-NEXT:    addiw a1, a1, 819
 ; RV64NOZBB-NEXT:    and a2, a0, a1
 ; RV64NOZBB-NEXT:    srli a0, a0, 2
 ; RV64NOZBB-NEXT:    and a0, a0, a1
 ; RV64NOZBB-NEXT:    add a0, a2, a0
 ; RV64NOZBB-NEXT:    srli a1, a0, 4
-; RV64NOZBB-NEXT:    add a0, a0, a1
-; RV64NOZBB-NEXT:    lui a1, 1
-; RV64NOZBB-NEXT:    addiw a1, a1, -241
-; RV64NOZBB-NEXT:    and a0, a0, a1
-; RV64NOZBB-NEXT:    slliw a1, a0, 8
-; RV64NOZBB-NEXT:    addw a0, a1, a0
-; RV64NOZBB-NEXT:    slli a0, a0, 51
-; RV64NOZBB-NEXT:    srli a0, a0, 59
+; RV64NOZBB-NEXT:    addw a0, a0, a1
+; RV64NOZBB-NEXT:    andi a1, a0, 15
+; RV64NOZBB-NEXT:    slli a0, a0, 52
+; RV64NOZBB-NEXT:    srli a0, a0, 60
+; RV64NOZBB-NEXT:    add a0, a1, a0
 ; RV64NOZBB-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: test_ctpop_i16:
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    zext.h a0, a0
 ; RV32ZBB-NEXT:    cpop a0, a0
 ; RV32ZBB-NEXT:    ret
 ;

llvm/test/CodeGen/X86/parity-vec.ll

 ; NOPOPCNT-NEXT:    shrl $2, %eax
 ; NOPOPCNT-NEXT:    andl $13107, %eax # imm = 0x3333
 ; NOPOPCNT-NEXT:    addl %ecx, %eax
 ; NOPOPCNT-NEXT:    movl %eax, %ecx
 ; NOPOPCNT-NEXT:    shrl $4, %ecx
 ; NOPOPCNT-NEXT:    addl %eax, %ecx
 ; NOPOPCNT-NEXT:    andl $3855, %ecx # imm = 0xF0F
 ; NOPOPCNT-NEXT:    movl %ecx, %eax
-; NOPOPCNT-NEXT:    shll $8, %eax
-; NOPOPCNT-NEXT:    addl %ecx, %eax
 ; NOPOPCNT-NEXT:    shrl $8, %eax
+; NOPOPCNT-NEXT:    addl %ecx, %eax
 ; NOPOPCNT-NEXT:    # kill: def $al killed $al killed $eax
 ; NOPOPCNT-NEXT:    retq
 ;
 ; POPCNT-LABEL: canonical_parity_noncanonical_pred:
 ; POPCNT:       # %bb.0:
 ; POPCNT-NEXT:    psllw $7, %xmm0
 ; POPCNT-NEXT:    pmovmskb %xmm0, %eax
 ; POPCNT-NEXT:    popcntl %eax, %eax

llvm/test/CodeGen/X86/popcnt.ll

 ; X86-NEXT:    shrl $2, %eax
 ; X86-NEXT:    andl $13107, %eax # imm = 0x3333
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    movl %eax, %ecx
 ; X86-NEXT:    shrl $4, %ecx
 ; X86-NEXT:    addl %eax, %ecx
 ; X86-NEXT:    andl $3855, %ecx # imm = 0xF0F
 ; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    shll $8, %eax
+; X86-NEXT:    shrl $8, %eax
 ; X86-NEXT:    addl %ecx, %eax
-; X86-NEXT:    movzbl %ah, %eax
+; X86-NEXT:    movzbl %al, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: cnt16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    shrl %eax
 ; X64-NEXT:    andl $21845, %eax # imm = 0x5555
 ; X64-NEXT:    subl %eax, %edi
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    andl $13107, %eax # imm = 0x3333
 ; X64-NEXT:    shrl $2, %edi
 ; X64-NEXT:    andl $13107, %edi # imm = 0x3333
 ; X64-NEXT:    addl %eax, %edi
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    shrl $4, %eax
 ; X64-NEXT:    addl %edi, %eax
 ; X64-NEXT:    andl $3855, %eax # imm = 0xF0F
 ; X64-NEXT:    movl %eax, %ecx
-; X64-NEXT:    shll $8, %ecx
+; X64-NEXT:    shrl $8, %ecx
 ; X64-NEXT:    addl %eax, %ecx
-; X64-NEXT:    movzbl %ch, %eax
+; X64-NEXT:    movzbl %cl, %eax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
 ;
 ; X86-POPCNT-LABEL: cnt16:
 ; X86-POPCNT:       # %bb.0:
 ; X86-POPCNT-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-POPCNT-NEXT:    popcntl %eax, %eax
 ; X86-POPCNT-NEXT:    # kill: def $ax killed $ax killed $eax

 ; X86-NEXT:    shrl $2, %eax
 ; X86-NEXT:    andl $13107, %eax # imm = 0x3333
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    movl %eax, %ecx
 ; X86-NEXT:    shrl $4, %ecx
 ; X86-NEXT:    addl %eax, %ecx
 ; X86-NEXT:    andl $3855, %ecx # imm = 0xF0F
 ; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    shll $8, %eax
+; X86-NEXT:    shrl $8, %eax
 ; X86-NEXT:    addl %ecx, %eax
-; X86-NEXT:    movzbl %ah, %eax
+; X86-NEXT:    movzbl %al, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: popcount_i16_zext:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    shrl %eax
 ; X64-NEXT:    andl $21845, %eax # imm = 0x5555
 ; X64-NEXT:    subl %eax, %edi
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    andl $13107, %eax # imm = 0x3333
 ; X64-NEXT:    shrl $2, %edi
 ; X64-NEXT:    andl $13107, %edi # imm = 0x3333
 ; X64-NEXT:    addl %eax, %edi
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    shrl $4, %eax
 ; X64-NEXT:    addl %edi, %eax
 ; X64-NEXT:    andl $3855, %eax # imm = 0xF0F
 ; X64-NEXT:    movl %eax, %ecx
-; X64-NEXT:    shll $8, %ecx
+; X64-NEXT:    shrl $8, %ecx
 ; X64-NEXT:    addl %eax, %ecx
-; X64-NEXT:    movzbl %ch, %eax
+; X64-NEXT:    movzbl %cl, %eax
 ; X64-NEXT:    retq
 ;
 ; X86-POPCNT-LABEL: popcount_i16_zext:
 ; X86-POPCNT:       # %bb.0:
 ; X86-POPCNT-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-POPCNT-NEXT:    popcntl %eax, %eax
 ; X86-POPCNT-NEXT:    retl
 ;