[TargetLowering] Only demand a rotation's modulo amount bits

[RKSimon]

Summary

ISD::ROTL/ROTR rotation values are guaranteed to act as a modulo amount, so for power-of-2 bitwidths we only need the lowest bits.

[RKSimon]

llvm/test/CodeGen/SystemZ/rot-01.ll

31

@uweigand Should this be 32 or 64?

llvm/test/CodeGen/SystemZ/rot-02.ll

7

@uweigand This comment doesn't seem to match the test - any suggestions?

[uweigand]

llvm/test/CodeGen/SystemZ/rot-01.ll

13

In order for this test to still test what it is intended to test (see below), this should probably be changed to 16 instead of 32 now.

31

This should be 32. The point of this test is that we actually need the AND, but we should use the most efficient instruction (when using the result of the AND only in a rotate, we can implement the AND via NILL instead of the NILF we'd otherwise require).

llvm/test/CodeGen/SystemZ/rot-02.ll

7

With these changes, only 5 bits count (instead of 6 bits) when operating on 32-bit variables. You should probably change the comment here to 5 bits, and then change the AND constant from 31 to 15 or so.

llvm/test/CodeGen/SystemZ/shift-04.ll

100–101

Now that we actually do mask the amount, the comment should be updated.

120

Now I'm wondering: why doesn't the masking also work here?

150

Or here?

160

Likewise.

llvm/test/CodeGen/SystemZ/shift-08.ll

101–102

Same comments as with shift-04.ll apply to this file.

[RKSimon]

Tweaked systemz tests based on feedback from @uweigand

[aykevl]

See my inline comment. I only checked rol8 on AVR but I think the same applies to ror8. Basically it seems like the transformation is correct but will make the operation unacceptably slower.

I'm not sure I can help you fix this, I'm still very new to backend development.

llvm/test/CodeGen/AVR/rot.ll

7

If I'm reading this IR, correctly, it does a rotate left just as the name implies.
I think the transformation here is correct in that it will still produce the same output, but without the mask the rotate may take a lot longer. I think the AVR assembly could be written as the following pseudocode (assume 8-bit unsigned integers everywhere):

// r24 = val
// r22 = amt
def rol8(r24, r22):
    r22 &= 7           // this mask is removed by this patch
    if r22 == 0:
        return r24     // return value (LBB0_2)
    while 1:
        r24 = (r24 << 1) | (r24 >> 7) // rotate r24 left by 1 (lsl, adc)
        r22 -= 1
        if r22 == 0:   // brne .LBB0_1
            return r24 // .LBB0_2, ret

So if you're calling rol8(x, 200) it will take 200 iterations to rotate a number.

[RKSimon]

llvm/test/CodeGen/AVR/rot.ll

7

I think this is a fault in the AVR backend - ROTL/ROTR/FSHL/FSHR all assume modulo amounts, so AVRTargetLowering::LowerShifts should mask accordingly. I'll add this to the patch shortly.

[RKSimon]

Added explicit masking to AVX rotation lowering.

[aykevl]

Thank you for the fix for AVR! It's unfamiliar code to me but it seems reasonable.

[lebedev.ri]

Looks good to me.

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
       Known.One |= Known2.One;
       Known.Zero |= Known2.Zero;
     }
     break;
   }
   case ISD::ROTL:
   case ISD::ROTR: {
     SDValue Op0 = Op.getOperand(0);
+    SDValue Op1 = Op.getOperand(1);
 
     // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
     if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
       return TLO.CombineTo(Op, Op0);
+
+    // For pow-2 bitwidths we only demand the bottom modulo amt bits.
+    if (isPowerOf2_32(BitWidth)) {
+      APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
+      if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
+                               Depth + 1))
+        return true;
+    }
     break;
   }
   case ISD::BITREVERSE: {
     SDValue Src = Op.getOperand(0);
     APInt DemandedSrcBits = DemandedBits.reverseBits();
     if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
                              Depth + 1))
       return true;

llvm/test/CodeGen/AArch64/funnel-shift-rot.ll

 ; CHECK-NEXT:    ret
   %f = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %z)
   ret i32 %f
 }
 
 define i64 @rotl_i64(i64 %x, i64 %z) {
 ; CHECK-LABEL: rotl_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    neg x8, x1
+; CHECK-NEXT:    neg w8, w1
 ; CHECK-NEXT:    ror x0, x0, x8
 ; CHECK-NEXT:    ret
   %f = call i64 @llvm.fshl.i64(i64 %x, i64 %x, i64 %z)
   ret i64 %f
 }
 
 ; Vector rotate.
 

llvm/test/CodeGen/AVR/rot.ll

 ; RUN: llc < %s -march=avr | FileCheck %s
 
 ; Bit rotation tests.
 
 ; CHECK-LABEL: rol8:
 define i8 @rol8(i8 %val, i8 %amt) {
-  ; CHECK:      andi r22, 7
-
-  ; CHECK-NEXT: cpi r22, 0
+  ; CHECK:      cpi r22, 0
   ; CHECK-NEXT: breq .LBB0_2
 
 ; CHECK-NEXT: .LBB0_1:
   ; CHECK-NEXT: lsl r24
   ; CHECK-NEXT: adc r24, r1
   ; CHECK-NEXT: subi r22, 1
   ; CHECK-NEXT: brne .LBB0_1
 
 ; CHECK-NEXT: .LBB0_2:
   ; CHECK-NEXT: ret
   %mod = urem i8 %amt, 8
 
   %inv = sub i8 8, %mod
   %parta = shl i8 %val, %mod
   %partb = lshr i8 %val, %inv
 
   %rotl = or i8 %parta, %partb
 
   ret i8 %rotl
 }
 
 
 ; CHECK-LABEL: ror8:
 define i8 @ror8(i8 %val, i8 %amt) {
-  ; CHECK:      andi r22, 7
-
-  ; CHECK-NEXT: cpi r22, 0
+  ; CHECK:      cpi r22, 0
   ; CHECK-NEXT: breq .LBB1_2
 
 ; CHECK-NEXT: .LBB1_1:
   ; CHECK-NEXT: lsr r24
   ; CHECK-NEXT: ldi r0, 0
   ; CHECK-NEXT: ror r0
   ; CHECK-NEXT: or r24, r0
   ; CHECK-NEXT: subi r22, 1

llvm/test/CodeGen/PowerPC/rotl-2.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- | FileCheck %s
 
 define i32 @rotl32(i32 %A, i8 %Amt) nounwind {
 ; CHECK-LABEL: rotl32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    clrlwi 4, 4, 24
 ; CHECK-NEXT:    rotlw 3, 3, 4
 ; CHECK-NEXT:    blr
   %shift.upgrd.1 = zext i8 %Amt to i32    ; <i32> [#uses=1]
   %B = shl i32 %A, %shift.upgrd.1   ; <i32> [#uses=1]
   %Amt2 = sub i8 32, %Amt   ; <i8> [#uses=1]
   %shift.upgrd.2 = zext i8 %Amt2 to i32   ; <i32> [#uses=1]
   %C = lshr i32 %A, %shift.upgrd.2    ; <i32> [#uses=1]
   %D = or i32 %B, %C    ; <i32> [#uses=1]
   ret i32 %D
 }
 
 define i32 @rotr32(i32 %A, i8 %Amt) nounwind {
 ; CHECK-LABEL: rotr32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    subfic 4, 4, 32
-; CHECK-NEXT:    clrlwi 4, 4, 24
 ; CHECK-NEXT:    rotlw 3, 3, 4
 ; CHECK-NEXT:    blr
   %shift.upgrd.3 = zext i8 %Amt to i32    ; <i32> [#uses=1]
   %B = lshr i32 %A, %shift.upgrd.3    ; <i32> [#uses=1]
   %Amt2 = sub i8 32, %Amt   ; <i8> [#uses=1]
   %shift.upgrd.4 = zext i8 %Amt2 to i32   ; <i32> [#uses=1]
   %C = shl i32 %A, %shift.upgrd.4   ; <i32> [#uses=1]
   %D = or i32 %B, %C    ; <i32> [#uses=1]

llvm/test/CodeGen/SystemZ/rot-01.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; Test shortening of NILL to NILF when the result is used as a rotate amount.
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
 ; Test 32-bit rotate.
 define i32 @f1(i32 %val, i32 %amt) {
 ; CHECK-LABEL: f1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    nill %r3, 31
+; CHECK-NEXT:    nill %r3, 15
 ; CHECK-NEXT:    rll %r2, %r2, 0(%r3)
 ; CHECK-NEXT:    br %r14
-  %mod = urem i32 %amt, 32
+  %mod = urem i32 %amt, 16
 
   %inv = sub i32 32, %mod
   %parta = shl i32 %val, %mod
   %partb = lshr i32 %val, %inv
 
   %rotl = or i32 %parta, %partb
 
   ret i32 %rotl
 }
 
 ; Test 64-bit rotate.
 define i64 @f2(i64 %val, i64 %amt) {
 ; CHECK-LABEL: f2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    nill %r3, 31
 ; CHECK-NEXT:    rllg %r2, %r2, 0(%r3)
 ; CHECK-NEXT:    br %r14
   %mod = urem i64 %amt, 32
 
   %inv = sub i64 64, %mod
   %parta = shl i64 %val, %mod
   %partb = lshr i64 %val, %inv
 
   %rotl = or i64 %parta, %partb
 
   ret i64 %rotl
 }

llvm/test/CodeGen/SystemZ/rot-02.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; Test removal of AND operations that don't affect last 6 bits of rotate amount
 ; operand.
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
-; Test that AND is not removed when some lower 6 bits are not set.
+; Test that AND is not removed when some lower 5 bits are not set.
 define i32 @f1(i32 %val, i32 %amt) {
 ; CHECK-LABEL: f1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    nill %r3, 31
+; CHECK-NEXT:    nill %r3, 15
 ; CHECK-NEXT:    rll %r2, %r2, 0(%r3)
 ; CHECK-NEXT:    br %r14
-  %and = and i32 %amt, 31
+  %and = and i32 %amt, 15
 
   %inv = sub i32 32, %and
   %parta = shl i32 %val, %and
   %partb = lshr i32 %val, %inv
 
   %rotl = or i32 %parta, %partb
 
   ret i32 %rotl

llvm/test/CodeGen/SystemZ/shift-04.ll

; CHECK-NEXT:    br %r14
   %addtrunc = trunc i64 %add to i32
   %subtrunc = trunc i64 %sub to i32
   %parta = shl i32 %a, %addtrunc
   %partb = lshr i32 %a, %subtrunc
   %or = or i32 %parta, %partb
   ret i32 %or
 }
 
-; Check shift amounts that have the largest in-range constant term.  We could
-; mask the amount instead.
+; Check shift amounts that have the largest in-range constant term, and then
+; mask the amount.
 define i32 @f8(i32 %a, i32 %amt) {
 ; CHECK-LABEL: f8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    rll %r2, %r2, 524287(%r3)
+; CHECK-NEXT:    rll %r2, %r2, -1(%r3)
 ; CHECK-NEXT:    br %r14
   %add = add i32 %amt, 524287
   %sub = sub i32 32, %add
   %parta = shl i32 %a, %add
   %partb = lshr i32 %a, %sub
   %or = or i32 %parta, %partb
   ret i32 %or
 }
 
 ; Check the next value up, which without masking must use a separate
 ; addition.
 define i32 @f9(i32 %a, i32 %amt) {
 ; CHECK-LABEL: f9:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    afi %r3, 524288
 ; CHECK-NEXT:    rll %r2, %r2, 0(%r3)
 ; CHECK-NEXT:    br %r14
   %add = add i32 %amt, 524288
   %sub = sub i32 32, %add
   %parta = shl i32 %a, %add
   %partb = lshr i32 %a, %sub
   %or = or i32 %parta, %partb
   ret i32 %or
; CHECK-NEXT:    br %r14
   ret i32 %or
 }
 
 ; Check the lowest value that can be subtracted from the shift amount.
 ; Again, we could mask the shift amount instead.
 define i32 @f11(i32 %a, i32 %amt) {
 ; CHECK-LABEL: f11:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    rll %r2, %r2, -524288(%r3)
 ; CHECK-NEXT:    br %r14
   %suba = sub i32 %amt, 524288
   %subb = sub i32 32, %suba
   %parta = shl i32 %a, %suba
   %partb = lshr i32 %a, %subb
   %or = or i32 %parta, %partb
   ret i32 %or
 }
 
-; Check the next value down, which without masking must use a separate
-; addition.
+; Check the next value down, masking the amount removes the addition.
 define i32 @f12(i32 %a, i32 %amt) {
 ; CHECK-LABEL: f12:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    afi %r3, -524289
-; CHECK-NEXT:    rll %r2, %r2, 0(%r3)
+; CHECK-NEXT:    rll %r2, %r2, -1(%r3)
 ; CHECK-NEXT:    br %r14
   %suba = sub i32 %amt, 524289
   %subb = sub i32 32, %suba
   %parta = shl i32 %a, %suba
   %partb = lshr i32 %a, %subb
   %or = or i32 %parta, %partb
   ret i32 %or
 }

llvm/test/CodeGen/SystemZ/shift-08.ll

; CHECK-NEXT:    br %r14
   %addext = zext i32 %add to i64
   %subext = zext i32 %sub to i64
   %parta = shl i64 %a, %addext
   %partb = lshr i64 %a, %subext
   %or = or i64 %parta, %partb
   ret i64 %or
 }
 
-; Check shift amounts that have the largest in-range constant term.  We could
-; mask the amount instead.
+; Check shift amounts that have the largest in-range constant term, and then
+; mask the amount.
 define i64 @f8(i64 %a, i64 %amt) {
 ; CHECK-LABEL: f8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    rllg %r2, %r2, 524287(%r3)
+; CHECK-NEXT:    rllg %r2, %r2, -1(%r3)
 ; CHECK-NEXT:    br %r14
   %add = add i64 %amt, 524287
   %sub = sub i64 64, %add
   %parta = shl i64 %a, %add
   %partb = lshr i64 %a, %sub
   %or = or i64 %parta, %partb
   ret i64 %or
 }
; CHECK-NEXT:    br %r14
   %suba = sub i64 %amt, 524288
   %subb = sub i64 64, %suba
   %parta = shl i64 %a, %suba
   %partb = lshr i64 %a, %subb
   %or = or i64 %parta, %partb
   ret i64 %or
 }
 
-; Check the next value down, which without masking must use a separate
-; addition.
+; Check the next value down, masking the amount removes the addition.
 define i64 @f12(i64 %a, i64 %amt) {
 ; CHECK-LABEL: f12:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    afi %r3, -524289
-; CHECK-NEXT:    rllg %r2, %r2, 0(%r3)
+; CHECK-NEXT:    rllg %r2, %r2, -1(%r3)
 ; CHECK-NEXT:    br %r14
   %suba = sub i64 %amt, 524289
   %subb = sub i64 64, %suba
   %parta = shl i64 %a, %suba
   %partb = lshr i64 %a, %subb
   %or = or i64 %parta, %partb
   ret i64 %or
 }

llvm/test/CodeGen/Thumb2/thumb2-ror.ll

 define i32 @f2(i32 %v, i32 %nbits) {
 ; CHECK-LABEL: f2:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    rors r0, r1
 ; CHECK-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: f2:
 ; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    movs r2, #31
-; THUMB1-NEXT:    ands r2, r1
-; THUMB1-NEXT:    rors r0, r2
+; THUMB1-NEXT:    rors r0, r1
 ; THUMB1-NEXT:    bx lr
 entry:
   %and = and i32 %nbits, 31
   %shr = lshr i32 %v, %and
   %sub = sub i32 32, %and
   %shl = shl i32 %v, %sub
   %or = or i32 %shl, %shr
   ret i32 %or
 }

llvm/test/CodeGen/X86/combine-rotates.ll

; CHECK-NEXT:    retq
   %9 = icmp eq i32 %1, 0
   %10 = select i1 %9, i32 %0, i32 %8
   ret i32 %10
 }
 
 define <4 x i32> @combine_vec_rot_select_zero(<4 x i32>, <4 x i32>) {
 ; SSE2-LABEL: combine_vec_rot_select_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [31,31,31,31]
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pslld $23, %xmm2
-; SSE2-NEXT:    paddd {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    cvttps2dq %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [31,31,31,31]
+; SSE2-NEXT:    pand %xmm1, %xmm3
+; SSE2-NEXT:    pslld $23, %xmm3
+; SSE2-NEXT:    paddd {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    cvttps2dq %xmm3, %xmm3
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pmuludq %xmm2, %xmm4
+; SSE2-NEXT:    pmuludq %xmm3, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,3,2,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
 ; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm4, %xmm3
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm4, %xmm2
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; XOP-LABEL: combine_vec_rot_select_zero:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm2
-; XOP-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; XOP-NEXT:    vprotd %xmm2, %xmm0, %xmm2
-; XOP-NEXT:    vpcomeqd %xmm3, %xmm1, %xmm1
-; XOP-NEXT:    vblendvps %xmm1, %xmm0, %xmm2, %xmm0
+; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vprotd %xmm1, %xmm0, %xmm3
+; XOP-NEXT:    vpcomeqd %xmm2, %xmm1, %xmm1
+; XOP-NEXT:    vblendvps %xmm1, %xmm0, %xmm3, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX2-LABEL: combine_vec_rot_select_zero:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpsllvd %xmm2, %xmm0, %xmm4
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
+; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm3
+; AVX2-NEXT:    vpsllvd %xmm3, %xmm0, %xmm4
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm5 = [32,32,32,32]
-; AVX2-NEXT:    vpsubd %xmm2, %xmm5, %xmm2
-; AVX2-NEXT:    vpsrlvd %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpor %xmm2, %xmm4, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vblendvps %xmm1, %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vpsubd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpsrlvd %xmm3, %xmm0, %xmm3
+; AVX2-NEXT:    vpor %xmm3, %xmm4, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vblendvps %xmm1, %xmm0, %xmm3, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: combine_vec_rot_select_zero:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm1, %xmm2
-; AVX512-NEXT:    vprolvd %xmm2, %xmm0, %xmm2
+; AVX512-NEXT:    vprolvd %xmm1, %xmm0, %xmm2
 ; AVX512-NEXT:    vptestnmd %xmm1, %xmm1, %k1
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovdqa %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %3 = and <4 x i32> %1, <i32 31, i32 31, i32 31, i32 31>
   %4 = shl <4 x i32> %0, %3
   %5 = sub <4 x i32> zeroinitializer, %1
   %6 = and <4 x i32> %5, <i32 31, i32 31, i32 31, i32 31>

 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; XOP-LABEL: rotate_demanded_bits_3:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpaddd %xmm1, %xmm1, %xmm1
-; XOP-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; XOP-NEXT:    vprotd %xmm1, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX2-LABEL: rotate_demanded_bits_3:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [30,30,30,30]
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
 ; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm2
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32]
 ; AVX2-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: rotate_demanded_bits_3:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm1, %xmm1
 ; AVX512-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %3 = shl <4 x i32> %1, <i32 1, i32 1, i32 1, i32 1>
   %4 = and <4 x i32> %3, <i32 30, i32 30, i32 30, i32 30>
   %5 = shl <4 x i32> %0, %4
   %6 = sub <4 x i32> zeroinitializer, %3
   %7 = and <4 x i32> %6, <i32 30, i32 30, i32 30, i32 30>
   %8 = lshr <4 x i32> %0, %7

llvm/test/CodeGen/X86/vector-fshl-rot-512.ll

 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512F-NEXT:    vpsrlw $4, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpandn %ymm4, %ymm5, %ymm4
 ; AVX512F-NEXT:    vpsllw $4, %ymm3, %ymm6
 ; AVX512F-NEXT:    vpand %ymm5, %ymm6, %ymm6
 ; AVX512F-NEXT:    vpor %ymm4, %ymm6, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $6, %ymm3, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512F-NEXT:    vpandn %ymm4, %ymm7, %ymm4
-; AVX512F-NEXT:    vpsllw $2, %ymm3, %ymm8
-; AVX512F-NEXT:    vpand %ymm7, %ymm8, %ymm8
-; AVX512F-NEXT:    vpor %ymm4, %ymm8, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512F-NEXT:    vpandn %ymm4, %ymm6, %ymm4
+; AVX512F-NEXT:    vpsllw $2, %ymm3, %ymm7
+; AVX512F-NEXT:    vpand %ymm6, %ymm7, %ymm7
+; AVX512F-NEXT:    vpor %ymm4, %ymm7, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $7, %ymm3, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm4, %ymm8, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm9
-; AVX512F-NEXT:    vpor %ymm4, %ymm9, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm8
+; AVX512F-NEXT:    vpor %ymm4, %ymm8, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm3
 ; AVX512F-NEXT:    vpandn %ymm3, %ymm5, %ymm3
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm4
 ; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vpand %ymm6, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm3
-; AVX512F-NEXT:    vpandn %ymm3, %ymm7, %ymm3
+; AVX512F-NEXT:    vpandn %ymm3, %ymm6, %ymm3
 ; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm4
-; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm6, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm3
-; AVX512F-NEXT:    vpand %ymm3, %ymm8, %ymm3
+; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
 ; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm4
 ; AVX512VL-NEXT:    vpsllw $4, %ymm3, %ymm5
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512VL-NEXT:    vpternlogq $226, %ymm4, %ymm6, %ymm5
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VL-NEXT:    vpand %ymm4, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw $6, %ymm3, %ymm5
-; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm7
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512VL-NEXT:    vpternlogq $226, %ymm5, %ymm8, %ymm7
+; AVX512VL-NEXT:    vpsrlw $6, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm4, %ymm7, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm7, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw $7, %ymm3, %ymm5
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512VL-NEXT:    vpand %ymm7, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm9
-; AVX512VL-NEXT:    vpor %ymm5, %ymm9, %ymm5
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw $7, %ymm3, %ymm4
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm8
+; AVX512VL-NEXT:    vpor %ymm4, %ymm8, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm5
-; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm6, %ymm5
-; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm6, %ymm4
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm5, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm8, %ymm4
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm7, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpand %ymm5, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
 ; AVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v64i8:

llvm/test/CodeGen/X86/vector-fshr-rot-512.ll

 ; AVX512F-NEXT:    vpsrlw $4, %ymm3, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpandn %ymm4, %ymm5, %ymm4
 ; AVX512F-NEXT:    vpsllw $4, %ymm3, %ymm6
 ; AVX512F-NEXT:    vpand %ymm5, %ymm6, %ymm6
 ; AVX512F-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; AVX512F-NEXT:    vpxor %xmm6, %xmm6, %xmm6
 ; AVX512F-NEXT:    vpsubb %ymm2, %ymm6, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $6, %ymm3, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512F-NEXT:    vpandn %ymm4, %ymm8, %ymm4
-; AVX512F-NEXT:    vpsllw $2, %ymm3, %ymm9
-; AVX512F-NEXT:    vpand %ymm8, %ymm9, %ymm9
-; AVX512F-NEXT:    vpor %ymm4, %ymm9, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512F-NEXT:    vpandn %ymm4, %ymm7, %ymm4
+; AVX512F-NEXT:    vpsllw $2, %ymm3, %ymm8
+; AVX512F-NEXT:    vpand %ymm7, %ymm8, %ymm8
+; AVX512F-NEXT:    vpor %ymm4, %ymm8, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $7, %ymm3, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm9 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm4, %ymm9, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm10
-; AVX512F-NEXT:    vpor %ymm4, %ymm10, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT:    vpand %ymm4, %ymm8, %ymm4
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm9
+; AVX512F-NEXT:    vpor %ymm4, %ymm9, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm3
 ; AVX512F-NEXT:    vpandn %ymm3, %ymm5, %ymm3
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm4
 ; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; AVX512F-NEXT:    vpsubb %ymm1, %ymm6, %ymm1
-; AVX512F-NEXT:    vpand %ymm7, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm3
-; AVX512F-NEXT:    vpandn %ymm3, %ymm8, %ymm3
+; AVX512F-NEXT:    vpandn %ymm3, %ymm7, %ymm3
 ; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm4
-; AVX512F-NEXT:    vpand %ymm4, %ymm8, %ymm4
+; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm3
-; AVX512F-NEXT:    vpand %ymm3, %ymm9, %ymm3
+; AVX512F-NEXT:    vpand %ymm3, %ymm8, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
 ; AVX512F-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm4
 ; AVX512VL-NEXT:    vpsllw $4, %ymm3, %ymm5
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512VL-NEXT:    vpternlogq $226, %ymm4, %ymm6, %ymm5
 ; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpsubb %ymm2, %ymm4, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VL-NEXT:    vpand %ymm7, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm3, %ymm5
-; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm8
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm9 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512VL-NEXT:    vpternlogq $226, %ymm5, %ymm9, %ymm8
+; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm7
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm5, %ymm8, %ymm7
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm8, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm7, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm3, %ymm5
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512VL-NEXT:    vpand %ymm5, %ymm8, %ymm5
-; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm10
-; AVX512VL-NEXT:    vpor %ymm5, %ymm10, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT:    vpand %ymm7, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm9
+; AVX512VL-NEXT:    vpor %ymm5, %ymm9, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm5
 ; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm6, %ymm5
 ; AVX512VL-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; AVX512VL-NEXT:    vpand %ymm7, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm5, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm9, %ymm4
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm8, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand %ymm3, %ymm8, %ymm3
+; AVX512VL-NEXT:    vpand %ymm7, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
 ; AVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v64i8: