[InstCombine] canonicalize cmp-of-bitcast-of-vector-cmp to use zero constant

[spatel]

Summary

We can invert a compare constant and compare predicate and preserve the logic as shown in this sampling:
https://alive2.llvm.org/ce/z/YAXbfs
(In theory, we could deal with non-all-ones/zero as well, but it doesn't seem worthwhile.)

I noticed this as a part of the x86 codegen difference in https://llvm.org/PR51259 - that ends up using "test" instead of "not + cmp" in that example.

This pattern also shows up in https://llvm.org/PR41312 and https://llvm.org/PR50798 .

[lebedev.ri]

The actual pattern here is: https://alive2.llvm.org/ce/z/GKG6Sy
Can you generalize this somewhat by making use of isFreeToInvert() on the bitcast's argument?

[spatel]

In D107170#2916900, @lebedev.ri wrote:

The actual pattern here is: https://alive2.llvm.org/ce/z/GKG6Sy
Can you generalize this somewhat by making use of isFreeToInvert() on the bitcast's argument?

Yes, that's better. I'll add some tests and then update here.

[spatel]

Patch updated:
Use isFreeToInvert for more general pattern match. I added some tests to exercise that, and it also gives a few more diffs in the existing PhaseOrdering test file via matching fcmp.

[lebedev.ri]

LG, thank you.

[lebedev.ri]

Hm, to be noted, this is still only a subpattern, with further generalization being: (https://alive2.llvm.org/ seems to be down)

define i1 @src(i8 %x, i8 %y) {
  %nx = xor i8 %x, -1
  %r = icmp eq i8 %nx, %y
  ret i1 %r
}

define i1 @tgt(i8 %x, i8 %y) {
  %ny = xor i8 %y, -1
  %r = icmp eq i8 %x, %ny
  ret i1 %r
}

----------------------------------------
define i1 @src(i8 %x, i8 %y) {
%0:
  %nx = xor i8 %x, 255
  %r = icmp eq i8 %nx, %y
  ret i1 %r
}
=>
define i1 @tgt(i8 %x, i8 %y) {
%0:
  %ny = xor i8 %y, 255
  %r = icmp eq i8 %x, %ny
  ret i1 %r
}
Transformation seems to be correct!

and

define i8 @src(<2 x i4> %x) {
  %s = bitcast <2 x i4> %x to i8
  %r = xor i8 %s, -1
  ret i8 %r
}

define i8 @tgt(<2 x i4> %x) {
  %nx = xor <2 x i4> %x, <i4 -1, i4 -1>
  %r = bitcast <2 x i4> %nx to i8
  ret i8 %r
}

----------------------------------------
define i8 @src(<2 x i4> %x) {
%0:
  %s = bitcast <2 x i4> %x to i8
  %r = xor i8 %s, 255
  ret i8 %r
}
=>
define i8 @tgt(<2 x i4> %x) {
%0:
  %nx = xor <2 x i4> %x, { 15, 15 }
  %r = bitcast <2 x i4> %nx to i8
  ret i8 %r
}
Transformation seems to be correct!

[spatel]

In D107170#2918427, @lebedev.ri wrote:

Hm, to be noted, this is still only a subpattern, with further generalization being: (https://alive2.llvm.org/ seems to be down)

Thanks! I'll put a TODO note on this. At first glance, I'm not sure what steps would be needed in the more generalized forms. We do have foldBitCastBitwiseLogic(), but there's a comment in there about changing ops from scalar <-> vector.

[lebedev.ri]

In D107170#2918476, @spatel wrote:

In D107170#2918427, @lebedev.ri wrote:

Hm, to be noted, this is still only a subpattern, with further generalization being: (https://alive2.llvm.org/ seems to be down)

Thanks! I'll put a TODO note on this. At first glance, I'm not sure what steps would be needed in the more generalized forms. We do have foldBitCastBitwiseLogic(), but there's a comment in there about changing ops from scalar <-> vector.

Mainly the next step is: there shouldn't be isFreeToInvert(), it should be InstCombineInverter (a.k.a InstCombineNegator, but for inversion)

llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp

if (TrueWhenGreaterThan)
       Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SGT,
                                                        OrigLHS, OrigRHS));
 
     return replaceInstUsesWith(Cmp, Cond);
   }
   return nullptr;
 }
 
-static Instruction *foldICmpBitCast(ICmpInst &Cmp,
-                                    InstCombiner::BuilderTy &Builder) {
+Instruction *InstCombinerImpl::foldICmpBitCast(ICmpInst &Cmp) {
   auto *Bitcast = dyn_cast<BitCastInst>(Cmp.getOperand(0));
   if (!Bitcast)
     return nullptr;
 
   ICmpInst::Predicate Pred = Cmp.getPredicate();
   Value *Op1 = Cmp.getOperand(1);
   Value *BCSrcOp = Bitcast->getOperand(0);
 
if (Bitcast->getType()->isPointerTy() &&
     // so eliminate it as well.
     if (auto *BC2 = dyn_cast<BitCastInst>(Op1))
       Op1 = BC2->getOperand(0);
 
     Op1 = Builder.CreateBitCast(Op1, BCSrcOp->getType());
     return new ICmpInst(Pred, BCSrcOp, Op1);
   }
 
+  const APInt *C;
+  if (!match(Cmp.getOperand(1), m_APInt(C)) ||
+      !Bitcast->getType()->isIntegerTy() ||
+      !Bitcast->getSrcTy()->isIntOrIntVectorTy())
+    return nullptr;
+
+  // If this is checking if all elements of a vector compare are set or not,
+  // invert the casted vector equality compare and test if all compare
+  // elements are clear or not. Compare against zero is generally easier for
+  // analysis and codegen.
+  // icmp eq/ne (bitcast (not X) to iN), -1 --> icmp eq/ne (bitcast X to iN), 0
+  // Example: are all elements equal? --> are zero elements not equal?
+  // TODO: Try harder to reduce compare of 2 freely invertible operands?
+  if (Cmp.isEquality() && C->isAllOnesValue() && Bitcast->hasOneUse() &&
+      isFreeToInvert(BCSrcOp, BCSrcOp->hasOneUse())) {
+    Type *ScalarTy = Bitcast->getType();
+    Value *Cast = Builder.CreateBitCast(Builder.CreateNot(BCSrcOp), ScalarTy);
+    return new ICmpInst(Pred, Cast, ConstantInt::getNullValue(ScalarTy));
+  }
+
   // Folding: icmp <pred> iN X, C
   //  where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN
   //    and C is a splat of a K-bit pattern
   //    and SC is a constant vector = <C', C', C', ..., C'>
   // Into:
   //   %E = extractelement <M x iK> %vec, i32 C'
   //   icmp <pred> iK %E, trunc(C)
-  const APInt *C;
-  if (!match(Cmp.getOperand(1), m_APInt(C)) ||
-      !Bitcast->getType()->isIntegerTy() ||
-      !Bitcast->getSrcTy()->isIntOrIntVectorTy())
-    return nullptr;
-
   Value *Vec;
   ArrayRef<int> Mask;
   if (match(BCSrcOp, m_Shuffle(m_Value(Vec), m_Undef(), m_Mask(Mask)))) {
     // Check whether every element of Mask is the same constant
     if (is_splat(Mask)) {
       auto *VecTy = cast<VectorType>(BCSrcOp->getType());
       auto *EltTy = cast<IntegerType>(VecTy->getElementType());
       if (C->isSplat(EltTy->getBitWidth())) {
if (Op0->getType()->isPointerTy() && I.isEquality()) {
     if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op0)))
       if (Instruction *New = foldAllocaCmp(I, Alloca, Op1))
         return New;
     if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op1)))
       if (Instruction *New = foldAllocaCmp(I, Alloca, Op0))
         return New;
   }
 
-  if (Instruction *Res = foldICmpBitCast(I, Builder))
+  if (Instruction *Res = foldICmpBitCast(I))
     return Res;
 
   // TODO: Hoist this above the min/max bailout.
   if (Instruction *R = foldICmpWithCastOp(I))
     return R;
 
   if (Instruction *Res = foldICmpWithMinMax(I))
     return Res;

llvm/lib/Transforms/InstCombine/InstCombineInternal.h

public:
 
   Instruction *foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp,
                                                  BinaryOperator *BO,
                                                  const APInt &C);
   Instruction *foldICmpIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II,
                                              const APInt &C);
   Instruction *foldICmpEqIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II,
                                                const APInt &C);
+  Instruction *foldICmpBitCast(ICmpInst &Cmp);
 
   // Helpers of visitSelectInst().
   Instruction *foldSelectExtConst(SelectInst &Sel);
   Instruction *foldSelectOpOp(SelectInst &SI, Instruction *TI, Instruction *FI);
   Instruction *foldSelectIntoOp(SelectInst &SI, Value *, Value *);
   Instruction *foldSPFofSPF(Instruction *Inner, SelectPatternFlavor SPF1,
                             Value *A, Value *B, Instruction &Outer,
                             SelectPatternFlavor SPF2, Value *C);

llvm/test/Transforms/InstCombine/icmp-vec.ll

 ;
   %cmp.ne = icmp ne <vscale x 2 x i64> %x, zeroinitializer
   %sel = select <vscale x 2 x i1> %cmp.ne, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> undef, i1 true, i32 0), <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i1> %falseval
   ret <vscale x 2 x i1> %sel
 }
 
 define i1 @eq_cast_eq-1(<2 x i4> %x, <2 x i4> %y) {
 ; CHECK-LABEL: @eq_cast_eq-1(
-; CHECK-NEXT:    [[IC:%.*]] = icmp eq <2 x i4> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = bitcast <2 x i1> [[IC]] to i2
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i2 [[B]], -1
+; CHECK-NEXT:    [[IC:%.*]] = icmp ne <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i1> [[IC]] to i2
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i2 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %ic = icmp eq <2 x i4> %x, %y
   %b = bitcast <2 x i1> %ic to i2
   %r = icmp eq i2 %b, -1
   ret i1 %r
 }
 
 define i1 @ne_cast_eq-1(<3 x i7> %x, <3 x i7> %y) {
 ; CHECK-LABEL: @ne_cast_eq-1(
-; CHECK-NEXT:    [[IC:%.*]] = icmp ne <3 x i7> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x i1> [[IC]] to i3
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i3 [[B]], -1
+; CHECK-NEXT:    [[IC:%.*]] = icmp eq <3 x i7> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <3 x i1> [[IC]] to i3
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i3 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %ic = icmp ne <3 x i7> %x, %y
   %b = bitcast <3 x i1> %ic to i3
   %r = icmp eq i3 %b, -1
   ret i1 %r
 }
 
 define i1 @eq_cast_ne-1(<2 x i7> %x, <2 x i7> %y) {
 ; CHECK-LABEL: @eq_cast_ne-1(
-; CHECK-NEXT:    [[IC:%.*]] = icmp eq <2 x i7> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = bitcast <2 x i1> [[IC]] to i2
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i2 [[B]], -1
+; CHECK-NEXT:    [[IC:%.*]] = icmp ne <2 x i7> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i1> [[IC]] to i2
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i2 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %ic = icmp eq <2 x i7> %x, %y
   %b = bitcast <2 x i1> %ic to i2
   %r = icmp ne i2 %b, -1
   ret i1 %r
 }
 
 define i1 @ne_cast_ne-1(<3 x i5> %x, <3 x i5> %y) {
 ; CHECK-LABEL: @ne_cast_ne-1(
-; CHECK-NEXT:    [[IC:%.*]] = icmp ne <3 x i5> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x i1> [[IC]] to i3
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i3 [[B]], -1
+; CHECK-NEXT:    [[IC:%.*]] = icmp eq <3 x i5> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <3 x i1> [[IC]] to i3
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i3 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %ic = icmp ne <3 x i5> %x, %y
   %b = bitcast <3 x i1> %ic to i3
   %r = icmp ne i3 %b, -1
   ret i1 %r
 }
 
 define i1 @ugt_cast_eq-1(<2 x i4> %x, <2 x i4> %y) {
 ; CHECK-LABEL: @ugt_cast_eq-1(
-; CHECK-NEXT:    [[IC:%.*]] = icmp ugt <2 x i4> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = bitcast <2 x i1> [[IC]] to i2
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i2 [[B]], -1
+; CHECK-NEXT:    [[IC:%.*]] = icmp ule <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i1> [[IC]] to i2
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i2 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %ic = icmp ugt <2 x i4> %x, %y
   %b = bitcast <2 x i1> %ic to i2
   %r = icmp eq i2 %b, -1
   ret i1 %r
 }
 
 define i1 @slt_cast_ne-1(<2 x i4> %x, <2 x i4> %y) {
 ; CHECK-LABEL: @slt_cast_ne-1(
-; CHECK-NEXT:    [[IC:%.*]] = icmp slt <2 x i4> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = bitcast <2 x i1> [[IC]] to i2
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i2 [[B]], -1
+; CHECK-NEXT:    [[IC:%.*]] = icmp sge <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i1> [[IC]] to i2
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i2 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %ic = icmp slt <2 x i4> %x, %y
   %b = bitcast <2 x i1> %ic to i2
   %r = icmp ne i2 %b, -1
   ret i1 %r
 }
 
 define i1 @ueq_cast_eq-1(<3 x float> %x, <3 x float> %y) {
 ; CHECK-LABEL: @ueq_cast_eq-1(
-; CHECK-NEXT:    [[FC:%.*]] = fcmp ueq <3 x float> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x i1> [[FC]] to i3
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i3 [[B]], -1
+; CHECK-NEXT:    [[FC:%.*]] = fcmp one <3 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <3 x i1> [[FC]] to i3
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i3 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %fc = fcmp ueq <3 x float> %x, %y
   %b = bitcast <3 x i1> %fc to i3
   %r = icmp eq i3 %b, -1
   ret i1 %r
 }
 
 define i1 @not_cast_ne-1(<3 x i1> %x) {
 ; CHECK-LABEL: @not_cast_ne-1(
-; CHECK-NEXT:    [[NOT:%.*]] = xor <3 x i1> [[X:%.*]], <i1 true, i1 true, i1 true>
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x i1> [[NOT]] to i3
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i3 [[B]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <3 x i1> [[X:%.*]] to i3
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i3 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %not = xor <3 x i1> %x, <i1 -1, i1 -1, i1 -1>
   %b = bitcast <3 x i1> %not to i3
   %r = icmp ne i3 %b, -1
   ret i1 %r
 }
 
 define i1 @not_cast_ne-1_uses(<3 x i2> %x, <3 x i2>* %p) {
 ; CHECK-LABEL: @not_cast_ne-1_uses(
 ; CHECK-NEXT:    [[NOT:%.*]] = xor <3 x i2> [[X:%.*]], <i2 -1, i2 -1, i2 -1>
 ; CHECK-NEXT:    store <3 x i2> [[NOT]], <3 x i2>* [[P:%.*]], align 4
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x i2> [[NOT]] to i6
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i6 [[B]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <3 x i2> [[X]] to i6
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i6 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %not = xor <3 x i2> %x, <i2 -1, i2 -1, i2 -1>
   store <3 x i2> %not, <3 x i2>* %p
   %b = bitcast <3 x i2> %not to i6
   %r = icmp ne i6 %b, -1
   ret i1 %r
 }
 
+; negative test - need equality pred on 2nd cmp
+
 define i1 @eq_cast_sgt-1(<3 x i4> %x, <3 x i4> %y) {
 ; CHECK-LABEL: @eq_cast_sgt-1(
 ; CHECK-NEXT:    [[IC:%.*]] = icmp eq <3 x i4> [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x i1> [[IC]] to i3
 ; CHECK-NEXT:    [[R:%.*]] = icmp sgt i3 [[B]], -1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %ic = icmp eq <3 x i4> %x, %y
   %b = bitcast <3 x i1> %ic to i3
   %r = icmp sgt i3 %b, -1
   ret i1 %r
 }
 
+; negative test - need all-ones constant on 2nd cmp
+
 define i1 @eq_cast_eq1(<2 x i4> %x, <2 x i4> %y) {
 ; CHECK-LABEL: @eq_cast_eq1(
 ; CHECK-NEXT:    [[IC:%.*]] = icmp eq <2 x i4> [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    [[B:%.*]] = bitcast <2 x i1> [[IC]] to i2
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i2 [[B]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %ic = icmp eq <2 x i4> %x, %y
   %b = bitcast <2 x i1> %ic to i2
   %r = icmp eq i2 %b, 1
   ret i1 %r
 }
 
+; negative test - extra use
+
 define i1 @eq_cast_eq-1_use1(<2 x i4> %x, <2 x i4> %y, <2 x i1>* %p) {
 ; CHECK-LABEL: @eq_cast_eq-1_use1(
 ; CHECK-NEXT:    [[IC:%.*]] = icmp sgt <2 x i4> [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    store <2 x i1> [[IC]], <2 x i1>* [[P:%.*]], align 2
 ; CHECK-NEXT:    [[B:%.*]] = bitcast <2 x i1> [[IC]] to i2
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i2 [[B]], -1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %ic = icmp sgt <2 x i4> %x, %y
   store <2 x i1> %ic, <2 x i1>* %p
   %b = bitcast <2 x i1> %ic to i2
   %r = icmp eq i2 %b, -1
   ret i1 %r
 }
 
+; negative test - extra use
+
 define i1 @eq_cast_eq-1_use2(<2 x i4> %x, <2 x i4> %y, i2* %p) {
 ; CHECK-LABEL: @eq_cast_eq-1_use2(
 ; CHECK-NEXT:    [[IC:%.*]] = icmp sgt <2 x i4> [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    [[B:%.*]] = bitcast <2 x i1> [[IC]] to i2
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i2* [[P:%.*]] to <2 x i1>*
 ; CHECK-NEXT:    store <2 x i1> [[IC]], <2 x i1>* [[TMP1]], align 1
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i2 [[B]], -1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %ic = icmp sgt <2 x i4> %x, %y
   %b = bitcast <2 x i1> %ic to i2
   store i2 %b, i2* %p
   %r = icmp eq i2 %b, -1
   ret i1 %r
 }

llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-logical.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -O2 -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64--"
 
 define float @test_merge_allof_v4sf(<4 x float> %t) {
 ; CHECK-LABEL: @test_merge_allof_v4sf(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[T_FR:%.*]] = freeze <4 x float> [[T:%.*]]
-; CHECK-NEXT:    [[TMP0:%.*]] = fcmp olt <4 x float> [[T_FR]], zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = fcmp uge <4 x float> [[T_FR]], zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i1> [[TMP0]] to i4
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i4 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i4 [[TMP1]], 0
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[RETURN:%.*]], label [[LOR_LHS_FALSE:%.*]]
 ; CHECK:       lor.lhs.false:
-; CHECK-NEXT:    [[TMP3:%.*]] = fcmp ogt <4 x float> [[T_FR]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp ule <4 x float> [[T_FR]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i1> [[TMP3]] to i4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i4 [[TMP4]], -1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i4 [[TMP4]], 0
 ; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[T_FR]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[SHIFT]], [[T_FR]]
 ; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x float> [[TMP6]], i32 0
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP5]], float 0.000000e+00, float [[ADD]]
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
 ; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[SPEC_SELECT]], [[LOR_LHS_FALSE]] ]
 ; CHECK-NEXT:    ret float [[RETVAL_0]]
return:
   %retval.0 = phi float [ 0.000000e+00, %if.then ], [ %add, %if.end ]
   ret float %retval.0
 }
 
 define float @test_separate_allof_v4sf(<4 x float> %t) {
 ; CHECK-LABEL: @test_separate_allof_v4sf(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[T_FR:%.*]] = freeze <4 x float> [[T:%.*]]
-; CHECK-NEXT:    [[TMP0:%.*]] = fcmp olt <4 x float> [[T_FR]], zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = fcmp uge <4 x float> [[T_FR]], zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i1> [[TMP0]] to i4
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i4 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i4 [[TMP1]], 0
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[RETURN:%.*]], label [[IF_END:%.*]]
 ; CHECK:       if.end:
-; CHECK-NEXT:    [[TMP3:%.*]] = fcmp ogt <4 x float> [[T_FR]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp ule <4 x float> [[T_FR]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i1> [[TMP3]] to i4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i4 [[TMP4]], -1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i4 [[TMP4]], 0
 ; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[T_FR]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[SHIFT]], [[T_FR]]
 ; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x float> [[TMP6]], i32 0
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP5]], float 0.000000e+00, float [[ADD]]
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
 ; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[SPEC_SELECT]], [[IF_END]] ]
 ; CHECK-NEXT:    ret float [[RETVAL_0]]
return:
   %retval.0 = phi float [ 0.000000e+00, %if.then ], [ 0.000000e+00, %if.then35 ], [ %add, %if.end36 ]
   ret float %retval.0
 }
 
 define float @test_merge_allof_v4si(<4 x i32> %t) {
 ; CHECK-LABEL: @test_merge_allof_v4si(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[T_FR:%.*]] = freeze <4 x i32> [[T:%.*]]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt <4 x i32> [[T_FR]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt <4 x i32> [[T_FR]], zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i1> [[TMP0]] to i4
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i4 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i4 [[TMP1]], 0
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[RETURN:%.*]], label [[LOR_LHS_FALSE:%.*]]
 ; CHECK:       lor.lhs.false:
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt <4 x i32> [[T_FR]], <i32 255, i32 255, i32 255, i32 255>
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt <4 x i32> [[T_FR]], <i32 256, i32 256, i32 256, i32 256>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i1> [[TMP3]] to i4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i4 [[TMP4]], -1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i4 [[TMP4]], 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[RETURN]], label [[IF_END:%.*]]
 ; CHECK:       if.end:
 ; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[T_FR]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[SHIFT]], [[T_FR]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[T_FR]], [[SHIFT]]
 ; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x i32> [[TMP6]], i32 0
 ; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[ADD]] to float
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
 ; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi float [ [[CONV]], [[IF_END]] ], [ 0.000000e+00, [[LOR_LHS_FALSE]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    ret float [[RETVAL_0]]
 ;
 entry:
return:
   %retval.0 = phi float [ 0.000000e+00, %if.then ], [ %conv, %if.end ]
   ret float %retval.0
 }
 
 define i32 @test_separate_allof_v4si(<4 x i32> %t) {
 ; CHECK-LABEL: @test_separate_allof_v4si(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[T_FR:%.*]] = freeze <4 x i32> [[T:%.*]]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt <4 x i32> [[T_FR]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt <4 x i32> [[T_FR]], zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i1> [[TMP0]] to i4
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i4 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i4 [[TMP1]], 0
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[RETURN:%.*]], label [[IF_END:%.*]]
 ; CHECK:       if.end:
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt <4 x i32> [[T_FR]], <i32 255, i32 255, i32 255, i32 255>
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt <4 x i32> [[T_FR]], <i32 256, i32 256, i32 256, i32 256>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i1> [[TMP3]] to i4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i4 [[TMP4]], -1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i4 [[TMP4]], 0
 ; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[T_FR]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[SHIFT]], [[T_FR]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[T_FR]], [[SHIFT]]
 ; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x i32> [[TMP6]], i32 0
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP5]], i32 0, i32 [[ADD]]
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
 ; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SPEC_SELECT]], [[IF_END]] ]
 ; CHECK-NEXT:    ret i32 [[RETVAL_0]]
 ;
 entry: