[X86] Support ANDNP combine through vector_shuffle

[e-kud]

Summary

Combine

and (vector_shuffle<Z,...,Z>
         (insert_vector_elt undef, (xor X, -1), Z), undef), Y
->
andnp (vector_shuffle<Z,...,Z>
           (insert_vector_elt undef, X, Z), undef), Y

[e-kud]

Guard bitcasts and pshufd with single use

[pengfei]

llvm/test/CodeGen/X86/combine-bitselect.ll

568 ↗

(On Diff #477323)

The name seems a bit confusing. It is not just a v4i32 version of bitselect_v4i64_broadcast_rrr. Better give a meaningful name. Besides, why don't use i64 type like others? And it's better to pre-commit the test to just show the diff in this patch.

[RKSimon]

llvm/test/CodeGen/X86/combine-bitselect.ll

615 ↗

(On Diff #477323)

This test would probably be better inside combine-and.ll - I've improved test coverage in rG6f11c395f5899a10cbcc823cfd6ef7b08525a17e

[e-kud]

llvm/test/CodeGen/X86/combine-bitselect.ll

615 ↗

(On Diff #477323)

Yes, thank you! I'll move the test to combine-and.ll. I'd like to check with both AVX512 (broadcast) and SSE (pshufd). So I'm going to add another RUN with AVX512F prefix. Should I rename existing CHECK to something like SSE42 or leave it as is?

[e-kud]

Address to comments and put tests into https://reviews.llvm.org/D138734

[e-kud]

Add peeking through bitcasts for BROADCAST as well.

[e-kud]

Rebased on precommit tests

[pengfei]

LGTM.

[RKSimon]

llvm/lib/Target/X86/X86ISelLowering.cpp

48260

Did you look at matching a generic ShuffleSDNode with a broadcast mask instead of PSHUFD?

[e-kud]

Propose more generic approach using vector_shuffle

[RKSimon]

llvm/lib/Target/X86/X86ISelLowering.cpp

48289

(style) Missing assert message

48293

256-bit vectors should be OK on AVX1 - but OK to just add a TODO if there's no current test coverage.

48300

(style) Use early-out instead of nested ifs (and move oneuse to the end as its the most expensive call)

if (!SVN || !SVN->isSplat() || !SVN->getOperand(1).isUndef() || !SVN->hasOneUse())
  return SDValue();

48325

(style) Missing assert message

[RKSimon]

Do we have any 256/512 bit test coverage?

[e-kud]

In D138521#3974292, @RKSimon wrote:

Do we have any 256/512 bit test coverage?

Unfortunately, no. I only tested locally with 256 bits.

I found out that the generic approach has a problem. If we have a 256 or 512 bit vector, then we can't generate ANDNP with SSE because there is no type for 256 bits. But after type-legalizer we have several ISD::AND that can be combined into ANDNP.
It works with BROADCAST/PSHUFD approach but performing the generic-combine before type-legalization prevents it.

I think about and (shuffle(not)) -> and(not(shuffle)) reordering before type-legalization and then after legalization we combine them into ANDNP. However I'm not sure about such two-step combine. What do you think?

[RKSimon]

SplitOpsAndApply might be able to help here

[e-kud]

Added vector splitting

[RKSimon]

LGTM with one minor comment - cheers

llvm/lib/Target/X86/X86ISelLowering.cpp

48242

You might need to bitcast this back to Src.getValueType() - its less likely for scalars but IsNOT can peek through bitcasts

[e-kud]

Rebased and added a bitcast of IsNOT.

[e-kud]

llvm/lib/Target/X86/X86ISelLowering.cpp

48242

Indeed, I added a test where a result of xor is bitcasted and got a failed assertion. Thank you!

[e-kud]

Rebased.

[RKSimon]

LGTM

llvm/lib/Target/X86/X86ISelLowering.cpp

 }
 
 // Match (xor X, -1) -> X.
 // Match extract_subvector(xor X, -1) -> extract_subvector(X).
 // Match concat_vectors(xor X, -1, xor Y, -1) -> concat_vectors(X, Y).
 static SDValue IsNOT(SDValue V, SelectionDAG &DAG) {
   V = peekThroughBitcasts(V);
   if (V.getOpcode() == ISD::XOR &&
-      ISD::isBuildVectorAllOnes(V.getOperand(1).getNode()))
+      (ISD::isBuildVectorAllOnes(V.getOperand(1).getNode()) ||
+       isAllOnesConstant(V.getOperand(1))))
     return V.getOperand(0);
   if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
       (isNullConstant(V.getOperand(1)) || V.getOperand(0).hasOneUse())) {
     if (SDValue Not = IsNOT(V.getOperand(0), DAG)) {
       Not = DAG.getBitcast(V.getOperand(0).getValueType(), Not);
       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Not), V.getValueType(),
                          Not, V.getOperand(1));
     }
if (VT == MVT::f32 || VT == MVT::f64 ||
       }
     }
   }
   return SDValue();
 }
 
 /// Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).
 static SDValue combineAndNotIntoANDNP(SDNode *N, SelectionDAG &DAG) {
-  assert(N->getOpcode() == ISD::AND);
+  assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDNP");
 
   MVT VT = N->getSimpleValueType(0);
   if (!VT.is128BitVector() && !VT.is256BitVector() && !VT.is512BitVector())
     return SDValue();
 
   SDValue X, Y;
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
 
-  auto GetNot = [&VT, &DAG](SDValue V) {
-    // Basic X = NOT(Y) detection.
-    if (SDValue Not = IsNOT(V, DAG))
-      return Not;
-    // Fold BROADCAST(NOT(Y)) -> BROADCAST(Y).
-    if (V.getOpcode() == X86ISD::VBROADCAST) {
-      SDValue Src = V.getOperand(0);
-      EVT SrcVT = Src.getValueType();
-      if (!SrcVT.isVector())
-        return SDValue();
-      if (SDValue Not = IsNOT(Src, DAG))
-        return DAG.getNode(X86ISD::VBROADCAST, SDLoc(V), VT,
-                           DAG.getBitcast(SrcVT, Not));
+  if (SDValue Not = IsNOT(N0, DAG)) {
+    X = Not;
+    Y = N1;
+  } else if (SDValue Not = IsNOT(N1, DAG)) {
+    X = Not;
+    Y = N0;
+  } else
+
+  X = DAG.getBitcast(VT, X);
+  Y = DAG.getBitcast(VT, Y);
+  return DAG.getNode(X86ISD::ANDNP, SDLoc(N), VT, X, Y);
+}
+
+/// Try to fold:
+///   and (vector_shuffle<Z,...,Z>
+///            (insert_vector_elt undef, (xor X, -1), Z), undef), Y
+///   ->
+///   andnp (vector_shuffle<Z,...,Z>
+///              (insert_vector_elt undef, X, Z), undef), Y
+static SDValue combineAndShuffleNot(SDNode *N, SelectionDAG &DAG,
+                                    const X86Subtarget &Subtarget) {
+  assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDNP");
+
+  EVT VT = N->getValueType(0);
+  // Do not split 256 and 512 bit vectors with SSE2 as they overwrite original
+  // value and require extra moves.
+  if (!((VT.is128BitVector() && Subtarget.hasSSE2()) ||
+        ((VT.is256BitVector() || VT.is512BitVector()) && Subtarget.hasAVX())))
+    return SDValue();
+
+  auto GetNot = [&DAG](SDValue V) {
+    auto *SVN = dyn_cast<ShuffleVectorSDNode>(peekThroughOneUseBitcasts(V));
+    // TODO: SVN->hasOneUse() is a strong condition. It can be relaxed if all
+    // end-users are ISD::AND including cases
+    // (and(extract_vector_element(SVN), Y)).
+    if (!SVN || !SVN->hasOneUse() || !SVN->isSplat() ||
+        !SVN->getOperand(1).isUndef()) {
+      return SDValue();
+    }
+    SDValue IVEN = SVN->getOperand(0);
+    if (IVEN.getOpcode() != ISD::INSERT_VECTOR_ELT ||
+        !IVEN.getOperand(0).isUndef() || !IVEN.hasOneUse())
+      return SDValue();
+    if (!isa<ConstantSDNode>(IVEN.getOperand(2)) ||
+        IVEN.getConstantOperandAPInt(2) != SVN->getSplatIndex())
+      return SDValue();
+    SDValue Src = IVEN.getOperand(1);
+    if (SDValue Not = IsNOT(Src, DAG)) {
+      SDValue NotSrc = DAG.getBitcast(Src.getValueType(), Not);
+      SDValue NotIVEN =
+          DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(IVEN), IVEN.getValueType(),
+                      IVEN.getOperand(0), NotSrc, IVEN.getOperand(2));
+      return DAG.getVectorShuffle(SVN->getValueType(0), SDLoc(SVN), NotIVEN,
+                                  SVN->getOperand(1), SVN->getMask());
     }
     return SDValue();
   };
 
+  SDValue X, Y;
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+
   if (SDValue Not = GetNot(N0)) {
     X = Not;
     Y = N1;
   } else if (SDValue Not = GetNot(N1)) {
     X = Not;
     Y = N0;
   } else
     return SDValue();
 
   X = DAG.getBitcast(VT, X);
   Y = DAG.getBitcast(VT, Y);
-  return DAG.getNode(X86ISD::ANDNP, SDLoc(N), VT, X, Y);
+  SDLoc DL(N);
+  // We do not split for SSE at all, but we need to split vectors for AVX1 and
+  // AVX2.
+  if (!Subtarget.useAVX512Regs() && VT.is512BitVector()) {
+    SDValue LoX, HiX;
+    std::tie(LoX, HiX) = splitVector(X, DAG, DL);
+    SDValue LoY, HiY;
+    std::tie(LoY, HiY) = splitVector(Y, DAG, DL);
+    EVT SplitVT = LoX.getValueType();
+    SDValue LoV = DAG.getNode(X86ISD::ANDNP, DL, SplitVT, {LoX, LoY});
+    SDValue HiV = DAG.getNode(X86ISD::ANDNP, DL, SplitVT, {HiX, HiY});
+    return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, {LoV, HiV});
+  }
+  return DAG.getNode(X86ISD::ANDNP, DL, VT, {X, Y});
 }
 
 // Try to widen AND, OR and XOR nodes to VT in order to remove casts around
 // logical operations, like in the example below.
 //   or (and (truncate x, truncate y)),
 //      (xor (truncate z, build_vector (constants)))
 // Given a target type \p VT, we generate
 //   or (and x, y), (xor z, zext(build_vector (constants)))
 // given x, y and z are of type \p VT. We can do so, if operands are either
 // truncates from VT types, the second operand is a vector of constants or can
 // be recursively promoted.
 static SDValue PromoteMaskArithmetic(SDNode *N, EVT VT, SelectionDAG &DAG,
                                      unsigned Depth) {
   // Limit recursion to avoid excessive compile times.
   if (Depth >= SelectionDAG::MaxRecursionDepth)
     return SDValue();
 
   if (N->getOpcode() != ISD::XOR && N->getOpcode() != ISD::AND &&
       N->getOpcode() != ISD::OR)
     return SDValue();
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   SDLoc DL(N);
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   if (!TLI.isOperationLegalOrPromote(N->getOpcode(), VT))
     return SDValue();
 
   if (SDValue NN0 = PromoteMaskArithmetic(N0.getNode(), VT, DAG, Depth + 1))
     N0 = NN0;
   else {
     // The Left side has to be a trunc.
     if (N0.getOpcode() != ISD::TRUNCATE)
       return SDValue();
 
     // The type of the truncated inputs.
     if (N0.getOperand(0).getValueType() != VT)
       return SDValue();
 
     N0 = N0.getOperand(0);
   }
 
   if (SDValue NN1 = PromoteMaskArithmetic(N1.getNode(), VT, DAG, Depth + 1))
     N1 = NN1;
   else {
     // The right side has to be a 'trunc' or a constant vector.
     bool RHSTrunc = N1.getOpcode() == ISD::TRUNCATE &&
                     N1.getOperand(0).getValueType() == VT;
     if (!RHSTrunc && !ISD::isBuildVectorOfConstantSDNodes(N1.getNode()))
       return SDValue();
 
     if (RHSTrunc)
       N1 = N1.getOperand(0);
     else
if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
     return R;
 
   if (SDValue R = combineBitOpWithShift(N, DAG))
     return R;
 
   if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget))
     return FPLogic;
 
+  if (SDValue R = combineAndShuffleNot(N, DAG, Subtarget))
+    return R;
+
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
   if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
     return R;
 
   if (SDValue R = combineAndNotIntoANDNP(N, DAG))
     return R;

llvm/test/CodeGen/X86/combine-and.ll

 ; AVX2-NEXT:    vmovq %rdi, %xmm2
 ; AVX2-NEXT:    vpbroadcastq %xmm2, %ymm2
 ; AVX2-NEXT:    vpand %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: neg_scalar_broadcast_v8i64_arg:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    notq %rdi
 ; AVX512-NEXT:    vpbroadcastq %rdi, %zmm1
-; AVX512-NEXT:    vpandq %zmm0, %zmm1, %zmm0
+; AVX512-NEXT:    vpandnq %zmm0, %zmm1, %zmm0
 ; AVX512-NEXT:    retq
   %1 = xor i64 %a0, -1
   %2 = insertelement <8 x i64> undef, i64 %1, i64 0
   %3 = shufflevector <8 x i64> %2, <8 x i64> poison, <8 x i32> zeroinitializer
   %4 = and <8 x i64> %3, %a1
   ret <8 x i64> %4
 }
 

 ; SSE-NEXT:    pand %xmm4, %xmm1
 ; SSE-NEXT:    pand %xmm4, %xmm2
 ; SSE-NEXT:    pand %xmm4, %xmm3
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: neg_scalar_broadcast_v8i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX1-NEXT:    notq %rdi
-; AVX1-NEXT:    vmovq %rdi, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[0,1,0,1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm2
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,1,0,1]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,3]
-; AVX1-NEXT:    vandpd %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vandpd %ymm1, %ymm2, %ymm1
+; AVX1-NEXT:    vmovq %rdi, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm2, %ymm2
+; AVX1-NEXT:    vandnpd %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    vandnpd %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: neg_scalar_broadcast_v8i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX2-NEXT:    notq %rdi
-; AVX2-NEXT:    vmovq %rdi, %xmm1
-; AVX2-NEXT:    vpbroadcastq %xmm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[0,1,0,0]
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,1,0,0]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,0,1,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vmovq %rdi, %xmm2
+; AVX2-NEXT:    vpbroadcastq %xmm2, %ymm2
+; AVX2-NEXT:    vpandn %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpandn %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: neg_scalar_broadcast_v8i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512-NEXT:    notq %rdi
 ; AVX512-NEXT:    vpbroadcastq %rdi, %zmm1
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [1,0,1,1,0,1,0,0]
 ; AVX512-NEXT:    vpermq %zmm0, %zmm2, %zmm0
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpandnq %zmm0, %zmm1, %zmm0
 ; AVX512-NEXT:    retq
   %1 = xor i64 %a0, -1
   %2 = insertelement <8 x i64> undef, i64 %1, i64 0
   %3 = shufflevector <8 x i64> %2, <8 x i64> poison, <8 x i32> zeroinitializer
   %4 = shufflevector <2 x i64> %a1, <2 x i64> poison, <8 x i32> <i32 1, i32 0, i32 1, i32 1, i32 0, i32 1, i32 0, i32 0>
   %5 = and <8 x i64> %4, %3
   ret <8 x i64> %5
 }
 
 define <4 x i64> @neg_scalar_broadcast_v4i64_arg(i64 %a0, <4 x i64> %a1) {
 ; SSE-LABEL: neg_scalar_broadcast_v4i64_arg:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    notq %rdi
 ; SSE-NEXT:    movq %rdi, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: neg_scalar_broadcast_v4i64_arg:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    notq %rdi
 ; AVX1-NEXT:    vmovq %rdi, %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
-; AVX1-NEXT:    vandps %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    vandnps %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: neg_scalar_broadcast_v4i64_arg:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    notq %rdi
 ; AVX2-NEXT:    vmovq %rdi, %xmm1
 ; AVX2-NEXT:    vpbroadcastq %xmm1, %ymm1
-; AVX2-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpandn %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: neg_scalar_broadcast_v4i64_arg:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    notq %rdi
 ; AVX512-NEXT:    vpbroadcastq %rdi, %ymm1
-; AVX512-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX512-NEXT:    vpandn %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %1 = xor i64 %a0, -1
   %2 = insertelement <4 x i64> undef, i64 %1, i64 0
   %3 = shufflevector <4 x i64> %2, <4 x i64> poison, <4 x i32> zeroinitializer
   %4 = and <4 x i64> %3, %a1
   ret <4 x i64> %4
 }
 
 define <4 x i64> @neg_scalar_broadcast_v4i64(i64 %a0, <2 x i64> %a1) {
 ; SSE-LABEL: neg_scalar_broadcast_v4i64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    notq %rdi
 ; SSE-NEXT:    movq %rdi, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,1,0,1]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: neg_scalar_broadcast_v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX1-NEXT:    notq %rdi
 ; AVX1-NEXT:    vmovq %rdi, %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,3]
-; AVX1-NEXT:    vandpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandnpd %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: neg_scalar_broadcast_v4i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX2-NEXT:    notq %rdi
 ; AVX2-NEXT:    vmovq %rdi, %xmm1
 ; AVX2-NEXT:    vpbroadcastq %xmm1, %ymm1
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,0,1,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpandn %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: neg_scalar_broadcast_v4i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512-NEXT:    notq %rdi
 ; AVX512-NEXT:    vpbroadcastq %rdi, %ymm1
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,0,1,1]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpandn %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %1 = xor i64 %a0, -1
   %2 = insertelement <4 x i64> undef, i64 %1, i64 0
   %3 = shufflevector <4 x i64> %2, <4 x i64> poison, <4 x i32> zeroinitializer
   %4 = shufflevector <2 x i64> %a1, <2 x i64> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 1>
   %5 = and <4 x i64> %4, %3
   ret <4 x i64> %5
 }
 
 define <2 x i64> @neg_scalar_broadcast_v2i64(i64 %a0, <2 x i64> %a1) {
 ; SSE-LABEL: neg_scalar_broadcast_v2i64:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    notq %rdi
 ; SSE-NEXT:    movq %rdi, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; SSE-NEXT:    pand %xmm1, %xmm0
+; SSE-NEXT:    pandn %xmm0, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: neg_scalar_broadcast_v2i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    notq %rdi
 ; AVX1-NEXT:    vmovq %rdi, %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; AVX1-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpandn %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: neg_scalar_broadcast_v2i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    notq %rdi
 ; AVX2-NEXT:    vmovq %rdi, %xmm1
 ; AVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpandn %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: neg_scalar_broadcast_v2i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    notq %rdi
 ; AVX512-NEXT:    vpbroadcastq %rdi, %xmm1
-; AVX512-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpandn %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %1 = xor i64 %a0, -1
   %2 = insertelement <2 x i64> undef, i64 %1, i64 0
   %3 = shufflevector <2 x i64> %2, <2 x i64> poison, <2 x i32> zeroinitializer
   %4 = and <2 x i64> %3, %a1
   ret <2 x i64> %4
 }
 

 ; SSE-NEXT:    movd %edi, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: neg_scalar_broadcast_v8i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    notl %edi
 ; AVX1-NEXT:    vmovd %edi, %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
-; AVX1-NEXT:    vandps %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    vandnps %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: neg_scalar_broadcast_v8i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    notl %edi
 ; AVX2-NEXT:    vmovd %edi, %xmm1
 ; AVX2-NEXT:    vpbroadcastd %xmm1, %ymm1
-; AVX2-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpandn %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: neg_scalar_broadcast_v8i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    notl %edi
 ; AVX512-NEXT:    vpbroadcastd %edi, %ymm1
-; AVX512-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX512-NEXT:    vpandn %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %1 = xor i32 %a0, -1
   %2 = insertelement <8 x i32> undef, i32 %1, i64 0
   %3 = shufflevector <8 x i32> %2, <8 x i32> poison, <8 x i32> zeroinitializer
   %4 = and <8 x i32> %3, %a1
   ret <8 x i32> %4
 }
 
 define <8 x i16> @neg_scalar_broadcast_v8i16(i16 %a0, <8 x i16> %a1) {
 ; SSE-LABEL: neg_scalar_broadcast_v8i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    notl %edi
 ; SSE-NEXT:    movd %edi, %xmm1
 ; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE-NEXT:    pand %xmm1, %xmm0
+; SSE-NEXT:    pandn %xmm0, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: neg_scalar_broadcast_v8i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    notl %edi
 ; AVX1-NEXT:    vmovd %edi, %xmm1
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; AVX1-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpandn %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: neg_scalar_broadcast_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    notl %edi
 ; AVX2-NEXT:    vmovd %edi, %xmm1
 ; AVX2-NEXT:    vpbroadcastw %xmm1, %xmm1
-; AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpandn %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: neg_scalar_broadcast_v8i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    notl %edi
 ; AVX512-NEXT:    vpbroadcastw %edi, %xmm1
-; AVX512-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpandn %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %1 = xor i16 %a0, -1
   %2 = insertelement <8 x i16> undef, i16 %1, i64 0
   %3 = shufflevector <8 x i16> %2, <8 x i16> poison, <8 x i32> zeroinitializer
   %4 = and <8 x i16> %3, %a1
   ret <8 x i16> %4
 }
 
 define <16 x i8> @neg_scalar_broadcast_v16i8(i8 %a0, <16 x i8> %a1) {
 ; SSE-LABEL: neg_scalar_broadcast_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    notb %dil
-; SSE-NEXT:    movzbl %dil, %eax
-; SSE-NEXT:    movd %eax, %xmm1
+; SSE-NEXT:    movd %edi, %xmm1
 ; SSE-NEXT:    pxor %xmm2, %xmm2
 ; SSE-NEXT:    pshufb %xmm2, %xmm1
-; SSE-NEXT:    pand %xmm1, %xmm0
+; SSE-NEXT:    pandn %xmm0, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: neg_scalar_broadcast_v16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    notb %dil
 ; AVX1-NEXT:    vmovd %edi, %xmm1
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpandn %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: neg_scalar_broadcast_v16i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    notb %dil
 ; AVX2-NEXT:    vmovd %edi, %xmm1
 ; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
-; AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpandn %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: neg_scalar_broadcast_v16i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    notb %dil
 ; AVX512-NEXT:    vpbroadcastb %edi, %xmm1
-; AVX512-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpandn %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %1 = xor i8 %a0, -1
   %2 = insertelement <16 x i8> undef, i8 %1, i64 0
   %3 = shufflevector <16 x i8> %2, <16 x i8> poison, <16 x i32> zeroinitializer
   %4 = and <16 x i8> %3, %a1
   ret <16 x i8> %4
 }
 

 ; AVX2-NEXT:    vmovd %edi, %xmm2
 ; AVX2-NEXT:    vpbroadcastb %xmm2, %ymm2
 ; AVX2-NEXT:    vpand %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: neg_scalar_broadcast_v64i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    notb %dil
 ; AVX512-NEXT:    vpbroadcastb %edi, %zmm1
-; AVX512-NEXT:    vpandq %zmm0, %zmm1, %zmm0
+; AVX512-NEXT:    vpandnq %zmm0, %zmm1, %zmm0
 ; AVX512-NEXT:    retq
   %1 = xor i8 %a0, -1
   %2 = insertelement <64 x i8> undef, i8 %1, i64 0
   %3 = shufflevector <64 x i8> %2, <64 x i8> poison, <64 x i32> zeroinitializer
   %4 = and <64 x i8> %3, %a1
   ret <64 x i8> %4
 }
 

 ; AVX2-NEXT:    vmovd %edi, %xmm2
 ; AVX2-NEXT:    vpbroadcastb %xmm2, %ymm2
 ; AVX2-NEXT:    vpand %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: neg_scalar_broadcast_v64i8_v8i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    notb %dil
 ; AVX512-NEXT:    vpbroadcastb %edi, %zmm1
-; AVX512-NEXT:    vpandq %zmm0, %zmm1, %zmm0
+; AVX512-NEXT:    vpandnq %zmm0, %zmm1, %zmm0
 ; AVX512-NEXT:    retq
   %1 = xor i8 %a0, -1
   %2 = insertelement <64 x i8> undef, i8 %1, i64 0
   %3 = shufflevector <64 x i8> %2, <64 x i8> poison, <64 x i32> zeroinitializer
   %4 = bitcast <64 x i8> %3 to <8 x i64>
   %5 = and <8 x i64> %4, %a1
   ret <8 x i64> %5
 }
 
 define <4 x i64> @neg_scalar_broadcast_v32i8_v4i64(i8 %a0, <4 x i64> %a1) {
 ; SSE-LABEL: neg_scalar_broadcast_v32i8_v4i64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    notb %dil
 ; SSE-NEXT:    movzbl %dil, %eax
 ; SSE-NEXT:    movd %eax, %xmm2
 ; SSE-NEXT:    pxor %xmm3, %xmm3
 ; SSE-NEXT:    pshufb %xmm3, %xmm2
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: neg_scalar_broadcast_v32i8_v4i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    notb %dil
 ; AVX1-NEXT:    vmovd %edi, %xmm1
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
-; AVX1-NEXT:    vandps %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    vandnps %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: neg_scalar_broadcast_v32i8_v4i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    notb %dil
 ; AVX2-NEXT:    vmovd %edi, %xmm1
 ; AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX2-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpandn %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: neg_scalar_broadcast_v32i8_v4i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    notb %dil
 ; AVX512-NEXT:    vpbroadcastb %edi, %ymm1
-; AVX512-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX512-NEXT:    vpandn %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %1 = xor i8 %a0, -1
   %2 = insertelement <32 x i8> undef, i8 %1, i64 0
   %3 = shufflevector <32 x i8> %2, <32 x i8> poison, <32 x i32> zeroinitializer
   %4 = bitcast <32 x i8> %3 to <4 x i64>
   %5 = and <4 x i64> %4, %a1
   ret <4 x i64> %5
 }
 
 define <2 x i64> @neg_scalar_broadcast_v16i8_v2i64(i8 %a0, <2 x i64> %a1) {
 ; SSE-LABEL: neg_scalar_broadcast_v16i8_v2i64:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    notb %dil
-; SSE-NEXT:    movzbl %dil, %eax
-; SSE-NEXT:    movd %eax, %xmm1
+; SSE-NEXT:    movd %edi, %xmm1
 ; SSE-NEXT:    pxor %xmm2, %xmm2
 ; SSE-NEXT:    pshufb %xmm2, %xmm1
-; SSE-NEXT:    pand %xmm1, %xmm0
+; SSE-NEXT:    pandn %xmm0, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: neg_scalar_broadcast_v16i8_v2i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    notb %dil
 ; AVX1-NEXT:    vmovd %edi, %xmm1
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpandn %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: neg_scalar_broadcast_v16i8_v2i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    notb %dil
 ; AVX2-NEXT:    vmovd %edi, %xmm1
 ; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
-; AVX2-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpandn %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: neg_scalar_broadcast_v16i8_v2i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    notb %dil
 ; AVX512-NEXT:    vpbroadcastb %edi, %xmm1
-; AVX512-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpandn %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %1 = xor i8 %a0, -1
   %2 = insertelement <16 x i8> undef, i8 %1, i64 0
   %3 = shufflevector <16 x i8> %2, <16 x i8> poison, <16 x i32> zeroinitializer
   %4 = bitcast <16 x i8> %3 to <2 x i64>
   %5 = and <2 x i64> %4, %a1
   ret <2 x i64> %5
 }
 
 define <4 x i64> @neg_scalar_broadcast_v8i32_v4i64(i32 %a0, <4 x i64> %a1) {
 ; SSE-LABEL: neg_scalar_broadcast_v8i32_v4i64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    notl %edi
 ; SSE-NEXT:    movd %edi, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: neg_scalar_broadcast_v8i32_v4i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    notl %edi
 ; AVX1-NEXT:    vmovd %edi, %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
-; AVX1-NEXT:    vandps %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    vandnps %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: neg_scalar_broadcast_v8i32_v4i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    notl %edi
 ; AVX2-NEXT:    vmovd %edi, %xmm1
 ; AVX2-NEXT:    vpbroadcastd %xmm1, %ymm1
-; AVX2-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpandn %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: neg_scalar_broadcast_v8i32_v4i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    notl %edi
 ; AVX512-NEXT:    vpbroadcastd %edi, %ymm1
-; AVX512-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX512-NEXT:    vpandn %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %1 = xor i32 %a0, -1
   %2 = insertelement <8 x i32> undef, i32 %1, i64 0
   %3 = shufflevector <8 x i32> %2, <8 x i32> poison, <8 x i32> zeroinitializer
   %4 = bitcast <8 x i32> %3 to <4 x i64>
   %5 = and <4 x i64> %4, %a1
   ret <4 x i64> %5
 }