[AArch64] Turn BIC creation into a DAG combine

[nikic]

Summary

This is an alternative to the AArch64 part of D59174. Instead of checking for and x, -1 specifically, this turns the and to bic transformation from custom lowering to a custom DAG combine. This gives a chance for generic DAG combines to apply first.

[efriedma]

I agree it doesn't really make sense to custom-lower this operation. That said, off the top of my head, I'm not sure why we're doing this at the DAG level at all, given the available instructions; it should be possible to pattern match (and x, (mvni imm)) during isel. (If there's some reason that doesn't work, a brief comment explaining it would be helpful.)

[nikic]

In D59187#1424052, @efriedma wrote:

I agree it doesn't really make sense to custom-lower this operation. That said, off the top of my head, I'm not sure why we're doing this at the DAG level at all, given the available instructions; it should be possible to pattern match (and x, (mvni imm)) during isel. (If there's some reason that doesn't work, a brief comment explaining it would be helpful.)

My guess would be that this would miss some cases that get lowered to (and x, (movi imm)), even though a representation using mvni would also be possible. The code for lowering constant build vectors first tries movi variants, then mvni, then movi with undef bits and finally mvni with undef bits.

[efriedma]

Okay, that's reasonable; please add a comment to the code.

[nikic]

Add comment on why this is not done in isel. Add updates to signed saturation tests added in the meantime.

[nikic]

Fix typo in comment.

[efriedma]

LGTM

llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h

private:
   SDValue LowerF128Call(SDValue Op, SelectionDAG &DAG,
                         RTLIB::Libcall Call) const;
   SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
-  SDValue LowerVectorAND(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerATOMIC_LOAD_AND(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerWindowsDYNAMIC_STACKALLOC(SDValue Op, SDValue Chain,

llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.cpp

for (unsigned im = (unsigned)ISD::PRE_INC;
     setIndexedStoreAction(im, MVT::f16, Legal);
   }
 
   // Trap.
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
 
   // We combine OR nodes for bitfield operations.
   setTargetDAGCombine(ISD::OR);
+  // Try to create BICs for vector ANDs.
+  setTargetDAGCombine(ISD::AND);
 
   // Vector add and sub nodes may conceal a high-half opportunity.
   // Also, try to fold ADD into CSINC/CSINV..
   setTargetDAGCombine(ISD::ADD);
   setTargetDAGCombine(ISD::SUB);
   setTargetDAGCombine(ISD::SRL);
   setTargetDAGCombine(ISD::XOR);
   setTargetDAGCombine(ISD::SINT_TO_FP);
void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) {
   setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
   setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
   setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
   setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
   setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
   setOperationAction(ISD::SRA, VT, Custom);
   setOperationAction(ISD::SRL, VT, Custom);
   setOperationAction(ISD::SHL, VT, Custom);
-  setOperationAction(ISD::AND, VT, Custom);
   setOperationAction(ISD::OR, VT, Custom);
   setOperationAction(ISD::SETCC, VT, Custom);
   setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
 
   setOperationAction(ISD::SELECT, VT, Expand);
   setOperationAction(ISD::SELECT_CC, VT, Expand);
   setOperationAction(ISD::VSELECT, VT, Expand);
   for (MVT InnerVT : MVT::all_valuetypes())
case ISD::SHL_PARTS:
     return LowerShiftLeftParts(Op, DAG);
   case ISD::SRL_PARTS:
   case ISD::SRA_PARTS:
     return LowerShiftRightParts(Op, DAG);
   case ISD::CTPOP:
     return LowerCTPOP(Op, DAG);
   case ISD::FCOPYSIGN:
     return LowerFCOPYSIGN(Op, DAG);
-  case ISD::AND:
-    return LowerVectorAND(Op, DAG);
   case ISD::OR:
     return LowerVectorOR(Op, DAG);
   case ISD::XOR:
     return LowerXOR(Op, DAG);
   case ISD::PREFETCH:
     return LowerPREFETCH(Op, DAG);
   case ISD::SINT_TO_FP:
   case ISD::UINT_TO_FP:
if (isAdvSIMDModImm) {
                                 DAG.getConstant(Value, dl, MVT::i32));
       return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
     }
   }
 
   return SDValue();
 }
 
-SDValue AArch64TargetLowering::LowerVectorAND(SDValue Op,
-                                              SelectionDAG &DAG) const {
-  SDValue LHS = Op.getOperand(0);
-  EVT VT = Op.getValueType();
-
-  BuildVectorSDNode *BVN =
-      dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode());
-  if (!BVN) {
-    // AND commutes, so try swapping the operands.
-    LHS = Op.getOperand(1);
-    BVN = dyn_cast<BuildVectorSDNode>(Op.getOperand(0).getNode());
-  }
-  if (!BVN)
-    return Op;
-
-  APInt DefBits(VT.getSizeInBits(), 0);
-  APInt UndefBits(VT.getSizeInBits(), 0);
-  if (resolveBuildVector(BVN, DefBits, UndefBits)) {
-    SDValue NewOp;
-
-    // We only have BIC vector immediate instruction, which is and-not.
-    DefBits = ~DefBits;
-    if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, Op, DAG,
-                                    DefBits, &LHS)) ||
-        (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, Op, DAG,
-                                    DefBits, &LHS)))
-      return NewOp;
-
-    UndefBits = ~UndefBits;
-    if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, Op, DAG,
-                                    UndefBits, &LHS)) ||
-        (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, Op, DAG,
-                                    UndefBits, &LHS)))
-      return NewOp;
-  }
-
-  // We can always fall back to a non-immediate AND.
-  return Op;
-}
-
 // Specialized code to quickly find if PotentialBVec is a BuildVector that
 // consists of only the same constant int value, returned in reference arg
 // ConstVal
 static bool isAllConstantBuildVector(const SDValue &PotentialBVec,
                                      uint64_t &ConstVal) {
   BuildVectorSDNode *Bvec = dyn_cast<BuildVectorSDNode>(PotentialBVec);
   if (!Bvec)
     return false;
if (SDValue Res = tryCombineToEXTR(N, DCI))
     return Res;
 
   if (SDValue Res = tryCombineToBSL(N, DCI))
     return Res;
 
   return SDValue();
 }
 
+static SDValue performANDCombine(SDNode *N,
+                                 TargetLowering::DAGCombinerInfo &DCI) {
+  SelectionDAG &DAG = DCI.DAG;
+  SDValue LHS = N->getOperand(0);
+  EVT VT = N->getValueType(0);
+  if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT))
+    return SDValue();
+
+  BuildVectorSDNode *BVN =
+      dyn_cast<BuildVectorSDNode>(N->getOperand(1).getNode());
+  if (!BVN)
+    return SDValue();
+
+  // AND does not accept an immediate, so check if we can use a BIC immediate
+  // instruction instead. We do this here instead of using a (and x, (mvni imm))
+  // pattern in isel, because some immediates may be lowered to the preferred
+  // (and x, (movi imm)) form, even though an mvni representation also exists.
+  APInt DefBits(VT.getSizeInBits(), 0);
+  APInt UndefBits(VT.getSizeInBits(), 0);
+  if (resolveBuildVector(BVN, DefBits, UndefBits)) {
+    SDValue NewOp;
+
+    DefBits = ~DefBits;
+    if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG,
+                                    DefBits, &LHS)) ||
+        (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG,
+                                    DefBits, &LHS)))
+      return NewOp;
+
+    UndefBits = ~UndefBits;
+    if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG,
+                                    UndefBits, &LHS)) ||
+        (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG,
+                                    UndefBits, &LHS)))
+      return NewOp;
+  }
+
+  return SDValue();
+}
+
 static SDValue performSRLCombine(SDNode *N,
                                  TargetLowering::DAGCombinerInfo &DCI) {
   SelectionDAG &DAG = DCI.DAG;
   EVT VT = N->getValueType(0);
   if (VT != MVT::i32 && VT != MVT::i64)
     return SDValue();
 
   // Canonicalize (srl (bswap i32 x), 16) to (rotr (bswap i32 x), 16), if the
case ISD::UINT_TO_FP:
     return performIntToFpCombine(N, DAG, Subtarget);
   case ISD::FP_TO_SINT:
   case ISD::FP_TO_UINT:
     return performFpToIntCombine(N, DAG, DCI, Subtarget);
   case ISD::FDIV:
     return performFDivCombine(N, DAG, DCI, Subtarget);
   case ISD::OR:
     return performORCombine(N, DCI, Subtarget);
+  case ISD::AND:
+    return performANDCombine(N, DCI);
   case ISD::SRL:
     return performSRLCombine(N, DCI);
   case ISD::INTRINSIC_WO_CHAIN:
     return performIntrinsicCombine(N, DCI, Subtarget);
   case ISD::ANY_EXTEND:
   case ISD::ZERO_EXTEND:
   case ISD::SIGN_EXTEND:
     return performExtendCombine(N, DCI, DAG);

llvm/trunk/test/CodeGen/AArch64/sadd_sat.ll

 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: vec:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v2.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
 ; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v4.4s, v2.4s, #0
-; CHECK-NEXT:    movi v3.4s, #128, lsl #24
-; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v4.4s
+; CHECK-NEXT:    cmge v5.4s, v2.4s, #0
 ; CHECK-NEXT:    cmlt v4.4s, v2.4s, #0
-; CHECK-NEXT:    bic v3.16b, v3.16b, v4.16b
-; CHECK-NEXT:    bic v4.4s, #128, lsl #24
+; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v5.4s
+; CHECK-NEXT:    mvni v3.4s, #128, lsl #24
+; CHECK-NEXT:    mvn v5.16b, v4.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    orr v3.16b, v4.16b, v3.16b
+; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
 ; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %tmp = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
   ret <4 x i32> %tmp;
 }

llvm/trunk/test/CodeGen/AArch64/sadd_sat_vec.ll

 }
 
 define <8 x i16> @v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; CHECK-LABEL: v8i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v2.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
 ; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v4.8h, v2.8h, #0
-; CHECK-NEXT:    movi v3.8h, #128, lsl #8
-; CHECK-NEXT:    cmeq v1.8h, v0.8h, v1.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v4.8h
+; CHECK-NEXT:    cmge v5.8h, v2.8h, #0
 ; CHECK-NEXT:    cmlt v4.8h, v2.8h, #0
-; CHECK-NEXT:    bic v3.16b, v3.16b, v4.16b
-; CHECK-NEXT:    bic v4.8h, #128, lsl #8
+; CHECK-NEXT:    cmeq v1.8h, v0.8h, v1.8h
+; CHECK-NEXT:    cmeq v0.8h, v0.8h, v5.8h
+; CHECK-NEXT:    mvni v3.8h, #128, lsl #8
+; CHECK-NEXT:    mvn v5.16b, v4.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    orr v3.16b, v4.16b, v3.16b
+; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
 ; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %z = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
   ret <8 x i16> %z
 }
 
 define <16 x i16> @v16i16(<16 x i16> %x, <16 x i16> %y) nounwind {
 ; CHECK-LABEL: v16i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v4.8h, v0.8h, v2.8h
+; CHECK-NEXT:    cmlt v16.8h, v4.8h, #0
+; CHECK-NEXT:    mvni v6.8h, #128, lsl #8
+; CHECK-NEXT:    add v7.8h, v1.8h, v3.8h
+; CHECK-NEXT:    mvn v17.16b, v16.16b
+; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmlt v16.8h, v7.8h, #0
+; CHECK-NEXT:    mvni v5.8h, #128, lsl #8
+; CHECK-NEXT:    mvn v17.16b, v16.16b
+; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
 ; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
 ; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    add v6.8h, v1.8h, v3.8h
 ; CHECK-NEXT:    cmge v16.8h, v4.8h, #0
-; CHECK-NEXT:    movi v5.8h, #128, lsl #8
 ; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
 ; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmlt v7.8h, v4.8h, #0
 ; CHECK-NEXT:    cmeq v2.8h, v0.8h, v2.8h
 ; CHECK-NEXT:    cmeq v0.8h, v0.8h, v16.8h
-; CHECK-NEXT:    cmge v16.8h, v6.8h, #0
+; CHECK-NEXT:    cmge v16.8h, v7.8h, #0
 ; CHECK-NEXT:    cmeq v3.8h, v1.8h, v3.8h
 ; CHECK-NEXT:    cmeq v1.8h, v1.8h, v16.8h
-; CHECK-NEXT:    bic v16.16b, v5.16b, v7.16b
-; CHECK-NEXT:    bic v7.8h, #128, lsl #8
-; CHECK-NEXT:    orr v7.16b, v7.16b, v16.16b
-; CHECK-NEXT:    cmlt v16.8h, v6.8h, #0
-; CHECK-NEXT:    bic v5.16b, v5.16b, v16.16b
-; CHECK-NEXT:    bic v16.8h, #128, lsl #8
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    orr v5.16b, v16.16b, v5.16b
 ; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
 ; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bsl v0.16b, v7.16b, v4.16b
-; CHECK-NEXT:    bsl v1.16b, v5.16b, v6.16b
+; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
+; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %x, <16 x i16> %y)
   ret <16 x i16> %z
 }
 
 define <32 x i16> @v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {
 ; CHECK-LABEL: v32i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v18.8h, v1.8h, v5.8h
+; CHECK-NEXT:    add v16.8h, v0.8h, v4.8h
+; CHECK-NEXT:    cmlt v24.8h, v16.8h, #0
+; CHECK-NEXT:    mvni v18.8h, #128, lsl #8
+; CHECK-NEXT:    add v19.8h, v1.8h, v5.8h
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v18.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmlt v24.8h, v19.8h, #0
+; CHECK-NEXT:    mvni v20.8h, #128, lsl #8
+; CHECK-NEXT:    add v21.8h, v2.8h, v6.8h
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmlt v24.8h, v21.8h, #0
+; CHECK-NEXT:    mvni v22.8h, #128, lsl #8
+; CHECK-NEXT:    add v23.8h, v3.8h, v7.8h
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmlt v24.8h, v23.8h, #0
+; CHECK-NEXT:    mvni v17.8h, #128, lsl #8
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmge v4.8h, v4.8h, #0
+; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
+; CHECK-NEXT:    cmge v24.8h, v16.8h, #0
 ; CHECK-NEXT:    cmge v5.8h, v5.8h, #0
 ; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    add v19.8h, v2.8h, v6.8h
-; CHECK-NEXT:    cmge v24.8h, v18.8h, #0
-; CHECK-NEXT:    add v16.8h, v0.8h, v4.8h
+; CHECK-NEXT:    cmeq v4.8h, v0.8h, v4.8h
+; CHECK-NEXT:    cmeq v0.8h, v0.8h, v24.8h
+; CHECK-NEXT:    cmge v24.8h, v19.8h, #0
 ; CHECK-NEXT:    cmge v6.8h, v6.8h, #0
 ; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    add v20.8h, v3.8h, v7.8h
 ; CHECK-NEXT:    cmeq v5.8h, v1.8h, v5.8h
 ; CHECK-NEXT:    cmeq v1.8h, v1.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v19.8h, #0
-; CHECK-NEXT:    movi v17.8h, #128, lsl #8
+; CHECK-NEXT:    cmge v24.8h, v21.8h, #0
 ; CHECK-NEXT:    cmge v7.8h, v7.8h, #0
 ; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmlt v21.8h, v16.8h, #0
 ; CHECK-NEXT:    cmeq v6.8h, v2.8h, v6.8h
 ; CHECK-NEXT:    cmeq v2.8h, v2.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v20.8h, #0
-; CHECK-NEXT:    cmge v4.8h, v4.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v22.8h, v16.8h, #0
-; CHECK-NEXT:    cmlt v23.8h, v18.8h, #0
+; CHECK-NEXT:    cmge v24.8h, v23.8h, #0
 ; CHECK-NEXT:    cmeq v7.8h, v3.8h, v7.8h
 ; CHECK-NEXT:    cmeq v3.8h, v3.8h, v24.8h
-; CHECK-NEXT:    bic v24.16b, v17.16b, v21.16b
-; CHECK-NEXT:    bic v21.8h, #128, lsl #8
-; CHECK-NEXT:    cmeq v4.8h, v0.8h, v4.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v22.8h
-; CHECK-NEXT:    cmlt v22.8h, v19.8h, #0
-; CHECK-NEXT:    orr v21.16b, v21.16b, v24.16b
-; CHECK-NEXT:    bic v24.16b, v17.16b, v23.16b
-; CHECK-NEXT:    bic v23.8h, #128, lsl #8
-; CHECK-NEXT:    orr v23.16b, v23.16b, v24.16b
-; CHECK-NEXT:    bic v24.16b, v17.16b, v22.16b
-; CHECK-NEXT:    bic v22.8h, #128, lsl #8
-; CHECK-NEXT:    orr v22.16b, v22.16b, v24.16b
-; CHECK-NEXT:    cmlt v24.8h, v20.8h, #0
-; CHECK-NEXT:    bic v17.16b, v17.16b, v24.16b
-; CHECK-NEXT:    bic v24.8h, #128, lsl #8
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    mvn v1.16b, v1.16b
 ; CHECK-NEXT:    mvn v2.16b, v2.16b
 ; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    orr v17.16b, v24.16b, v17.16b
 ; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
 ; CHECK-NEXT:    and v1.16b, v5.16b, v1.16b
 ; CHECK-NEXT:    and v2.16b, v6.16b, v2.16b
 ; CHECK-NEXT:    and v3.16b, v7.16b, v3.16b
-; CHECK-NEXT:    bsl v0.16b, v21.16b, v16.16b
-; CHECK-NEXT:    bsl v1.16b, v23.16b, v18.16b
-; CHECK-NEXT:    bsl v2.16b, v22.16b, v19.16b
-; CHECK-NEXT:    bsl v3.16b, v17.16b, v20.16b
+; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
+; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
+; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
+; CHECK-NEXT:    bsl v3.16b, v17.16b, v23.16b
 ; CHECK-NEXT:    ret
   %z = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %x, <32 x i16> %y)
   ret <32 x i16> %z
 }
 
 define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v8i8:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    mov v1.h[2], w8
 ; CHECK-NEXT:    mov v0.h[3], w9
 ; CHECK-NEXT:    mov v1.h[3], w10
 ; CHECK-NEXT:    shl v1.4h, v1.4h, #8
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #8
 ; CHECK-NEXT:    add v3.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
 ; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v4.4h, v3.4h, #0
-; CHECK-NEXT:    movi v2.4h, #128, lsl #8
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v4.4h
+; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
 ; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    bic v2.8b, v2.8b, v4.8b
-; CHECK-NEXT:    bic v4.4h, #128, lsl #8
+; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
+; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
+; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
+; CHECK-NEXT:    mvn v5.8b, v4.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    orr v2.8b, v4.8b, v2.8b
+; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
 ; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
 ; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    str s0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <4 x i8>, <4 x i8>* %px
   %y = load <4 x i8>, <4 x i8>* %py

 ; CHECK-NEXT:    fmov s2, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
 ; CHECK-NEXT:    mov v2.s[1], w11
 ; CHECK-NEXT:    shl v2.2s, v2.2s, #24
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #24
 ; CHECK-NEXT:    add v3.2s, v0.2s, v2.2s
 ; CHECK-NEXT:    cmge v2.2s, v2.2s, #0
 ; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v4.2s, v3.2s, #0
-; CHECK-NEXT:    movi v1.2s, #128, lsl #24
-; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v4.2s
+; CHECK-NEXT:    cmge v5.2s, v3.2s, #0
 ; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    bic v1.8b, v1.8b, v4.8b
-; CHECK-NEXT:    bic v4.2s, #128, lsl #24
+; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
+; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
+; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
+; CHECK-NEXT:    mvn v5.8b, v4.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    orr v1.8b, v4.8b, v1.8b
+; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
 ; CHECK-NEXT:    and v0.8b, v2.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #24
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
 ; CHECK-NEXT:    strb w8, [x2, #1]
 ; CHECK-NEXT:    strb w9, [x2]
 ; CHECK-NEXT:    ret
   %x = load <2 x i8>, <2 x i8>* %px
   %y = load <2 x i8>, <2 x i8>* %py
   %z = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %x, <2 x i8> %y)
   store <2 x i8> %z, <2 x i8>* %pz
   ret void
 }
 
 define void @v4i16(<4 x i16>* %px, <4 x i16>* %py, <4 x i16>* %pz) nounwind {
 ; CHECK-LABEL: v4i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    movi v2.4h, #128, lsl #8
+; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
 ; CHECK-NEXT:    add v3.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
 ; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v4.4h
+; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
 ; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    bic v2.8b, v2.8b, v4.8b
-; CHECK-NEXT:    bic v4.4h, #128, lsl #8
+; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
+; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
+; CHECK-NEXT:    mvn v5.8b, v4.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    orr v2.8b, v4.8b, v2.8b
+; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
 ; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    str d0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <4 x i16>, <4 x i16>* %px
   %y = load <4 x i16>, <4 x i16>* %py
   %z = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> %x, <4 x i16> %y)
   store <4 x i16> %z, <4 x i16>* %pz

 ; CHECK-NEXT:    fmov s2, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
 ; CHECK-NEXT:    mov v2.s[1], w11
 ; CHECK-NEXT:    shl v2.2s, v2.2s, #16
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #16
 ; CHECK-NEXT:    add v3.2s, v0.2s, v2.2s
 ; CHECK-NEXT:    cmge v2.2s, v2.2s, #0
 ; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v4.2s, v3.2s, #0
-; CHECK-NEXT:    movi v1.2s, #128, lsl #24
-; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v4.2s
+; CHECK-NEXT:    cmge v5.2s, v3.2s, #0
 ; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    bic v1.8b, v1.8b, v4.8b
-; CHECK-NEXT:    bic v4.2s, #128, lsl #24
+; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
+; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
+; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
+; CHECK-NEXT:    mvn v5.8b, v4.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    orr v1.8b, v4.8b, v1.8b
+; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
 ; CHECK-NEXT:    and v0.8b, v2.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #16
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
 ; CHECK-NEXT:    strh w8, [x2, #2]
 ; CHECK-NEXT:    strh w9, [x2]
 ; CHECK-NEXT:    ret

 ; CHECK-NEXT:    ret
   %z = call <12 x i8> @llvm.sadd.sat.v12i8(<12 x i8> %x, <12 x i8> %y)
   ret <12 x i8> %z
 }
 
 define void @v12i16(<12 x i16>* %px, <12 x i16>* %py, <12 x i16>* %pz) nounwind {
 ; CHECK-LABEL: v12i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q2, q0, [x0]
-; CHECK-NEXT:    ldp q3, q1, [x1]
-; CHECK-NEXT:    movi v4.8h, #128, lsl #8
-; CHECK-NEXT:    add v5.8h, v0.8h, v1.8h
+; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q3, q2, [x1]
+; CHECK-NEXT:    mvni v5.8h, #128, lsl #8
+; CHECK-NEXT:    mvni v4.8h, #128, lsl #8
+; CHECK-NEXT:    add v6.8h, v1.8h, v2.8h
+; CHECK-NEXT:    cmlt v16.8h, v6.8h, #0
+; CHECK-NEXT:    add v7.8h, v0.8h, v3.8h
+; CHECK-NEXT:    mvn v17.16b, v16.16b
+; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmlt v16.8h, v7.8h, #0
+; CHECK-NEXT:    mvn v17.16b, v16.16b
+; CHECK-NEXT:    bsl v4.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
 ; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    add v6.8h, v2.8h, v3.8h
-; CHECK-NEXT:    cmge v16.8h, v5.8h, #0
+; CHECK-NEXT:    cmge v16.8h, v6.8h, #0
 ; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    cmlt v7.8h, v5.8h, #0
-; CHECK-NEXT:    cmeq v1.8h, v0.8h, v1.8h
+; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
+; CHECK-NEXT:    cmeq v2.8h, v1.8h, v2.8h
+; CHECK-NEXT:    cmeq v1.8h, v1.8h, v16.8h
+; CHECK-NEXT:    cmge v16.8h, v7.8h, #0
+; CHECK-NEXT:    cmeq v3.8h, v0.8h, v3.8h
 ; CHECK-NEXT:    cmeq v0.8h, v0.8h, v16.8h
-; CHECK-NEXT:    cmge v16.8h, v6.8h, #0
-; CHECK-NEXT:    cmeq v3.8h, v2.8h, v3.8h
-; CHECK-NEXT:    cmeq v2.8h, v2.8h, v16.8h
-; CHECK-NEXT:    bic v16.16b, v4.16b, v7.16b
-; CHECK-NEXT:    bic v7.8h, #128, lsl #8
-; CHECK-NEXT:    orr v7.16b, v7.16b, v16.16b
-; CHECK-NEXT:    cmlt v16.8h, v6.8h, #0
-; CHECK-NEXT:    bic v4.16b, v4.16b, v16.16b
-; CHECK-NEXT:    bic v16.8h, #128, lsl #8
+; CHECK-NEXT:    mvn v1.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    orr v4.16b, v16.16b, v4.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v2.16b
-; CHECK-NEXT:    bsl v0.16b, v7.16b, v5.16b
-; CHECK-NEXT:    bsl v1.16b, v4.16b, v6.16b
-; CHECK-NEXT:    str q1, [x2]
-; CHECK-NEXT:    str d0, [x2, #16]
+; CHECK-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-NEXT:    and v0.16b, v3.16b, v0.16b
+; CHECK-NEXT:    bsl v1.16b, v5.16b, v6.16b
+; CHECK-NEXT:    bsl v0.16b, v4.16b, v7.16b
+; CHECK-NEXT:    str q0, [x2]
+; CHECK-NEXT:    str d1, [x2, #16]
 ; CHECK-NEXT:    ret
   %x = load <12 x i16>, <12 x i16>* %px
   %y = load <12 x i16>, <12 x i16>* %py
   %z = call <12 x i16> @llvm.sadd.sat.v12i16(<12 x i16> %x, <12 x i16> %y)
   store <12 x i16> %z, <12 x i16>* %pz
   ret void
 }
 
; CHECK-NEXT:    ret
   ret void
 }
 
 define void @v1i16(<1 x i16>* %px, <1 x i16>* %py, <1 x i16>* %pz) nounwind {
 ; CHECK-LABEL: v1i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    ldr h1, [x1]
-; CHECK-NEXT:    movi v2.4h, #128, lsl #8
+; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
 ; CHECK-NEXT:    add v3.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
 ; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v4.4h
+; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
 ; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    bic v2.8b, v2.8b, v4.8b
-; CHECK-NEXT:    bic v4.4h, #128, lsl #8
+; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
+; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
+; CHECK-NEXT:    mvn v5.8b, v4.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    orr v2.8b, v4.8b, v2.8b
+; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
 ; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    str h0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <1 x i16>, <1 x i16>* %px
   %y = load <1 x i16>, <1 x i16>* %py
   %z = call <1 x i16> @llvm.sadd.sat.v1i16(<1 x i16> %x, <1 x i16> %y)
   store <1 x i16> %z, <1 x i16>* %pz

 }
 
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v2.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    cmge v1.2s, v1.2s, #0
 ; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v4.2s, v2.2s, #0
-; CHECK-NEXT:    movi v3.2s, #128, lsl #24
-; CHECK-NEXT:    cmeq v1.2s, v0.2s, v1.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v4.2s
+; CHECK-NEXT:    cmge v5.2s, v2.2s, #0
 ; CHECK-NEXT:    cmlt v4.2s, v2.2s, #0
-; CHECK-NEXT:    bic v3.8b, v3.8b, v4.8b
-; CHECK-NEXT:    bic v4.2s, #128, lsl #24
+; CHECK-NEXT:    cmeq v1.2s, v0.2s, v1.2s
+; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
+; CHECK-NEXT:    mvni v3.2s, #128, lsl #24
+; CHECK-NEXT:    mvn v5.8b, v4.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    orr v3.8b, v4.8b, v3.8b
+; CHECK-NEXT:    bsl v3.8b, v4.8b, v5.8b
 ; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v3.8b, v2.8b
 ; CHECK-NEXT:    ret
   %z = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
   ret <2 x i32> %z
 }
 
 define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: v4i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v2.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
 ; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v4.4s, v2.4s, #0
-; CHECK-NEXT:    movi v3.4s, #128, lsl #24
-; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v4.4s
+; CHECK-NEXT:    cmge v5.4s, v2.4s, #0
 ; CHECK-NEXT:    cmlt v4.4s, v2.4s, #0
-; CHECK-NEXT:    bic v3.16b, v3.16b, v4.16b
-; CHECK-NEXT:    bic v4.4s, #128, lsl #24
+; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v5.4s
+; CHECK-NEXT:    mvni v3.4s, #128, lsl #24
+; CHECK-NEXT:    mvn v5.16b, v4.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    orr v3.16b, v4.16b, v3.16b
+; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
 ; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %z = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
   ret <4 x i32> %z
 }
 
 define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; CHECK-LABEL: v8i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v4.4s, v0.4s, v2.4s
+; CHECK-NEXT:    cmlt v16.4s, v4.4s, #0
+; CHECK-NEXT:    mvni v6.4s, #128, lsl #24
+; CHECK-NEXT:    add v7.4s, v1.4s, v3.4s
+; CHECK-NEXT:    mvn v17.16b, v16.16b
+; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmlt v16.4s, v7.4s, #0
+; CHECK-NEXT:    mvni v5.4s, #128, lsl #24
+; CHECK-NEXT:    mvn v17.16b, v16.16b
+; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
 ; CHECK-NEXT:    cmge v2.4s, v2.4s, #0
 ; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    add v6.4s, v1.4s, v3.4s
 ; CHECK-NEXT:    cmge v16.4s, v4.4s, #0
-; CHECK-NEXT:    movi v5.4s, #128, lsl #24
 ; CHECK-NEXT:    cmge v3.4s, v3.4s, #0
 ; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmlt v7.4s, v4.4s, #0
 ; CHECK-NEXT:    cmeq v2.4s, v0.4s, v2.4s
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, v16.4s
-; CHECK-NEXT:    cmge v16.4s, v6.4s, #0
+; CHECK-NEXT:    cmge v16.4s, v7.4s, #0
 ; CHECK-NEXT:    cmeq v3.4s, v1.4s, v3.4s
 ; CHECK-NEXT:    cmeq v1.4s, v1.4s, v16.4s
-; CHECK-NEXT:    bic v16.16b, v5.16b, v7.16b
-; CHECK-NEXT:    bic v7.4s, #128, lsl #24
-; CHECK-NEXT:    orr v7.16b, v7.16b, v16.16b
-; CHECK-NEXT:    cmlt v16.4s, v6.4s, #0
-; CHECK-NEXT:    bic v5.16b, v5.16b, v16.16b
-; CHECK-NEXT:    bic v16.4s, #128, lsl #24
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    orr v5.16b, v16.16b, v5.16b
 ; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
 ; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bsl v0.16b, v7.16b, v4.16b
-; CHECK-NEXT:    bsl v1.16b, v5.16b, v6.16b
+; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
+; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
 ; CHECK-NEXT:    ret
   %z = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
   ret <8 x i32> %z
 }
 
 define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 ; CHECK-LABEL: v16i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v18.4s, v1.4s, v5.4s
+; CHECK-NEXT:    add v16.4s, v0.4s, v4.4s
+; CHECK-NEXT:    cmlt v24.4s, v16.4s, #0
+; CHECK-NEXT:    mvni v18.4s, #128, lsl #24
+; CHECK-NEXT:    add v19.4s, v1.4s, v5.4s
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v18.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmlt v24.4s, v19.4s, #0
+; CHECK-NEXT:    mvni v20.4s, #128, lsl #24
+; CHECK-NEXT:    add v21.4s, v2.4s, v6.4s
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmlt v24.4s, v21.4s, #0
+; CHECK-NEXT:    mvni v22.4s, #128, lsl #24
+; CHECK-NEXT:    add v23.4s, v3.4s, v7.4s
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmlt v24.4s, v23.4s, #0
+; CHECK-NEXT:    mvni v17.4s, #128, lsl #24
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmge v4.4s, v4.4s, #0
+; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
+; CHECK-NEXT:    cmge v24.4s, v16.4s, #0
 ; CHECK-NEXT:    cmge v5.4s, v5.4s, #0
 ; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    add v19.4s, v2.4s, v6.4s
-; CHECK-NEXT:    cmge v24.4s, v18.4s, #0
-; CHECK-NEXT:    add v16.4s, v0.4s, v4.4s
+; CHECK-NEXT:    cmeq v4.4s, v0.4s, v4.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v24.4s
+; CHECK-NEXT:    cmge v24.4s, v19.4s, #0
 ; CHECK-NEXT:    cmge v6.4s, v6.4s, #0
 ; CHECK-NEXT:    cmge v2.4s, v2.4s, #0
-; CHECK-NEXT:    add v20.4s, v3.4s, v7.4s
 ; CHECK-NEXT:    cmeq v5.4s, v1.4s, v5.4s
 ; CHECK-NEXT:    cmeq v1.4s, v1.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v19.4s, #0
-; CHECK-NEXT:    movi v17.4s, #128, lsl #24
+; CHECK-NEXT:    cmge v24.4s, v21.4s, #0
 ; CHECK-NEXT:    cmge v7.4s, v7.4s, #0
 ; CHECK-NEXT:    cmge v3.4s, v3.4s, #0
-; CHECK-NEXT:    cmlt v21.4s, v16.4s, #0
 ; CHECK-NEXT:    cmeq v6.4s, v2.4s, v6.4s
 ; CHECK-NEXT:    cmeq v2.4s, v2.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v20.4s, #0
-; CHECK-NEXT:    cmge v4.4s, v4.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v22.4s, v16.4s, #0
-; CHECK-NEXT:    cmlt v23.4s, v18.4s, #0
+; CHECK-NEXT:    cmge v24.4s, v23.4s, #0
 ; CHECK-NEXT:    cmeq v7.4s, v3.4s, v7.4s
 ; CHECK-NEXT:    cmeq v3.4s, v3.4s, v24.4s
-; CHECK-NEXT:    bic v24.16b, v17.16b, v21.16b
-; CHECK-NEXT:    bic v21.4s, #128, lsl #24
-; CHECK-NEXT:    cmeq v4.4s, v0.4s, v4.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v22.4s
-; CHECK-NEXT:    cmlt v22.4s, v19.4s, #0
-; CHECK-NEXT:    orr v21.16b, v21.16b, v24.16b
-; CHECK-NEXT:    bic v24.16b, v17.16b, v23.16b
-; CHECK-NEXT:    bic v23.4s, #128, lsl #24
-; CHECK-NEXT:    orr v23.16b, v23.16b, v24.16b
-; CHECK-NEXT:    bic v24.16b, v17.16b, v22.16b
-; CHECK-NEXT:    bic v22.4s, #128, lsl #24
-; CHECK-NEXT:    orr v22.16b, v22.16b, v24.16b
-; CHECK-NEXT:    cmlt v24.4s, v20.4s, #0
-; CHECK-NEXT:    bic v17.16b, v17.16b, v24.16b
-; CHECK-NEXT:    bic v24.4s, #128, lsl #24
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    mvn v1.16b, v1.16b
 ; CHECK-NEXT:    mvn v2.16b, v2.16b
 ; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    orr v17.16b, v24.16b, v17.16b
 ; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
 ; CHECK-NEXT:    and v1.16b, v5.16b, v1.16b
 ; CHECK-NEXT:    and v2.16b, v6.16b, v2.16b
 ; CHECK-NEXT:    and v3.16b, v7.16b, v3.16b
-; CHECK-NEXT:    bsl v0.16b, v21.16b, v16.16b
-; CHECK-NEXT:    bsl v1.16b, v23.16b, v18.16b
-; CHECK-NEXT:    bsl v2.16b, v22.16b, v19.16b
-; CHECK-NEXT:    bsl v3.16b, v17.16b, v20.16b
+; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
+; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
+; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
+; CHECK-NEXT:    bsl v3.16b, v17.16b, v23.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
   ret <16 x i32> %z
 }
 
 define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: v2i64:
 ; CHECK:       // %bb.0:

llvm/trunk/test/CodeGen/AArch64/sat-add.ll

 
 define <16 x i8> @unsigned_sat_constant_v16i8_using_cmp_sum(<16 x i8> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v16i8_using_cmp_sum:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.16b, #42
 ; CHECK-NEXT:    add v1.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    cmhi v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <16 x i8> %x, <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
   %c = icmp ugt <16 x i8> %x, %a
   %r = select <16 x i1> %c, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %a
   ret <16 x i8> %r
 }
 
 define <16 x i8> @unsigned_sat_constant_v16i8_using_cmp_notval(<16 x i8> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v16i8_using_cmp_notval:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.16b, #42
 ; CHECK-NEXT:    movi v2.16b, #213
 ; CHECK-NEXT:    add v1.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    cmhi v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <16 x i8> %x, <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
   %c = icmp ugt <16 x i8> %x, <i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43>
   %r = select <16 x i1> %c, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %a
   ret <16 x i8> %r
 }
 

 
 define <8 x i16> @unsigned_sat_constant_v8i16_using_cmp_sum(<8 x i16> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v8i16_using_cmp_sum:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.8h, #42
 ; CHECK-NEXT:    add v1.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    cmhi v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <8 x i16> %x, <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
   %c = icmp ugt <8 x i16> %x, %a
   %r = select <8 x i1> %c, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %a
   ret <8 x i16> %r
 }
 
 define <8 x i16> @unsigned_sat_constant_v8i16_using_cmp_notval(<8 x i16> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v8i16_using_cmp_notval:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.8h, #42
 ; CHECK-NEXT:    mvni v2.8h, #42
 ; CHECK-NEXT:    add v1.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    cmhi v0.8h, v0.8h, v2.8h
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <8 x i16> %x, <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
   %c = icmp ugt <8 x i16> %x, <i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43>
   %r = select <8 x i1> %c, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %a
   ret <8 x i16> %r
 }
 

 
 define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_sum(<4 x i32> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v4i32_using_cmp_sum:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #42
 ; CHECK-NEXT:    add v1.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    cmhi v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
   %c = icmp ugt <4 x i32> %x, %a
   %r = select <4 x i1> %c, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %a
   ret <4 x i32> %r
 }
 
 define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_notval(<4 x i32> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #42
 ; CHECK-NEXT:    mvni v2.4s, #42
 ; CHECK-NEXT:    add v1.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    cmhi v0.4s, v0.4s, v2.4s
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
   %c = icmp ugt <4 x i32> %x, <i32 -43, i32 -43, i32 -43, i32 -43>
   %r = select <4 x i1> %c, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %a
   ret <4 x i32> %r
 }
 

 define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_sum(<2 x i64> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v2i64_using_cmp_sum:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #42
 ; CHECK-NEXT:    dup v1.2d, x8
 ; CHECK-NEXT:    add v1.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    cmhi v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <2 x i64> %x, <i64 42, i64 42>
   %c = icmp ugt <2 x i64> %x, %a
   %r = select <2 x i1> %c, <2 x i64> <i64 -1, i64 -1>, <2 x i64> %a
   ret <2 x i64> %r
 }
 
 define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_notval(<2 x i64> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v2i64_using_cmp_notval:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #42
 ; CHECK-NEXT:    mov x9, #-43
 ; CHECK-NEXT:    dup v1.2d, x8
 ; CHECK-NEXT:    dup v2.2d, x9
 ; CHECK-NEXT:    add v1.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    cmhi v0.2d, v0.2d, v2.2d
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <2 x i64> %x, <i64 42, i64 42>
   %c = icmp ugt <2 x i64> %x, <i64 -43, i64 -43>
   %r = select <2 x i1> %c, <2 x i64> <i64 -1, i64 -1>, <2 x i64> %a
   ret <2 x i64> %r
 }
 

 }
 
 define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_sum(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v16i8_using_cmp_sum:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v1.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    cmhi v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <16 x i8> %x, %y
   %c = icmp ugt <16 x i8> %x, %a
   %r = select <16 x i1> %c, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %a
   ret <16 x i8> %r
 }
 
 define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_notval(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v16i8_using_cmp_notval:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mvn v2.16b, v1.16b
 ; CHECK-NEXT:    add v1.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    cmhi v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %noty = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
   %a = add <16 x i8> %x, %y
   %c = icmp ugt <16 x i8> %x, %noty
   %r = select <16 x i1> %c, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %a
   ret <16 x i8> %r
 }

 }
 
 define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_sum(<8 x i16> %x, <8 x i16> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v8i16_using_cmp_sum:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v1.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    cmhi v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <8 x i16> %x, %y
   %c = icmp ugt <8 x i16> %x, %a
   %r = select <8 x i1> %c, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %a
   ret <8 x i16> %r
 }
 
 define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_notval(<8 x i16> %x, <8 x i16> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v8i16_using_cmp_notval:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mvn v2.16b, v1.16b
 ; CHECK-NEXT:    add v1.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    cmhi v0.8h, v0.8h, v2.8h
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %noty = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
   %a = add <8 x i16> %x, %y
   %c = icmp ugt <8 x i16> %x, %noty
   %r = select <8 x i1> %c, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %a
   ret <8 x i16> %r
 }

 }
 
 define <4 x i32> @unsigned_sat_variable_v4i32_using_cmp_sum(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v4i32_using_cmp_sum:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v1.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    cmhi v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <4 x i32> %x, %y
   %c = icmp ugt <4 x i32> %x, %a
   %r = select <4 x i1> %c, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %a
   ret <4 x i32> %r
 }
 
 define <4 x i32> @unsigned_sat_variable_v4i32_using_cmp_notval(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v4i32_using_cmp_notval:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mvn v2.16b, v1.16b
 ; CHECK-NEXT:    add v1.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    cmhi v0.4s, v0.4s, v2.4s
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %noty = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
   %a = add <4 x i32> %x, %y
   %c = icmp ugt <4 x i32> %x, %noty
   %r = select <4 x i1> %c, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %a
   ret <4 x i32> %r
 }

 }
 
 define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_sum(<2 x i64> %x, <2 x i64> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v2i64_using_cmp_sum:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v1.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    cmhi v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <2 x i64> %x, %y
   %c = icmp ugt <2 x i64> %x, %a
   %r = select <2 x i1> %c, <2 x i64> <i64 -1, i64 -1>, <2 x i64> %a
   ret <2 x i64> %r
 }
 
 define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_notval(<2 x i64> %x, <2 x i64> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v2i64_using_cmp_notval:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mvn v2.16b, v1.16b
 ; CHECK-NEXT:    add v1.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    cmhi v0.2d, v0.2d, v2.2d
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %noty = xor <2 x i64> %y, <i64 -1, i64 -1>
   %a = add <2 x i64> %x, %y
   %c = icmp ugt <2 x i64> %x, %noty
   %r = select <2 x i1> %c, <2 x i64> <i64 -1, i64 -1>, <2 x i64> %a
   ret <2 x i64> %r
 }
 

llvm/trunk/test/CodeGen/AArch64/ssub_sat.ll

 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: vec:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v2.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
 ; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v4.4s, v2.4s, #0
-; CHECK-NEXT:    movi v3.4s, #128, lsl #24
-; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v4.4s
+; CHECK-NEXT:    cmge v5.4s, v2.4s, #0
 ; CHECK-NEXT:    cmlt v4.4s, v2.4s, #0
-; CHECK-NEXT:    bic v3.16b, v3.16b, v4.16b
-; CHECK-NEXT:    bic v4.4s, #128, lsl #24
+; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v5.4s
+; CHECK-NEXT:    mvni v3.4s, #128, lsl #24
+; CHECK-NEXT:    mvn v5.16b, v4.16b
 ; CHECK-NEXT:    mvn v1.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    orr v3.16b, v4.16b, v3.16b
+; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
 ; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %tmp = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
   ret <4 x i32> %tmp;
 }

llvm/trunk/test/CodeGen/AArch64/ssub_sat_vec.ll

 }
 
 define <8 x i16> @v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; CHECK-LABEL: v8i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v2.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
 ; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v4.8h, v2.8h, #0
-; CHECK-NEXT:    movi v3.8h, #128, lsl #8
-; CHECK-NEXT:    cmeq v1.8h, v0.8h, v1.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v4.8h
+; CHECK-NEXT:    cmge v5.8h, v2.8h, #0
 ; CHECK-NEXT:    cmlt v4.8h, v2.8h, #0
-; CHECK-NEXT:    bic v3.16b, v3.16b, v4.16b
-; CHECK-NEXT:    bic v4.8h, #128, lsl #8
+; CHECK-NEXT:    cmeq v1.8h, v0.8h, v1.8h
+; CHECK-NEXT:    cmeq v0.8h, v0.8h, v5.8h
+; CHECK-NEXT:    mvni v3.8h, #128, lsl #8
+; CHECK-NEXT:    mvn v5.16b, v4.16b
 ; CHECK-NEXT:    mvn v1.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    orr v3.16b, v4.16b, v3.16b
+; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
 ; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %z = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
   ret <8 x i16> %z
 }
 
 define <16 x i16> @v16i16(<16 x i16> %x, <16 x i16> %y) nounwind {
 ; CHECK-LABEL: v16i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v4.8h, v0.8h, v2.8h
+; CHECK-NEXT:    cmlt v16.8h, v4.8h, #0
+; CHECK-NEXT:    mvni v6.8h, #128, lsl #8
+; CHECK-NEXT:    sub v7.8h, v1.8h, v3.8h
+; CHECK-NEXT:    mvn v17.16b, v16.16b
+; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmlt v16.8h, v7.8h, #0
+; CHECK-NEXT:    mvni v5.8h, #128, lsl #8
+; CHECK-NEXT:    mvn v17.16b, v16.16b
+; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
 ; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
 ; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    sub v6.8h, v1.8h, v3.8h
 ; CHECK-NEXT:    cmge v16.8h, v4.8h, #0
-; CHECK-NEXT:    movi v5.8h, #128, lsl #8
 ; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
 ; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmlt v7.8h, v4.8h, #0
 ; CHECK-NEXT:    cmeq v2.8h, v0.8h, v2.8h
 ; CHECK-NEXT:    cmeq v0.8h, v0.8h, v16.8h
-; CHECK-NEXT:    cmge v16.8h, v6.8h, #0
+; CHECK-NEXT:    cmge v16.8h, v7.8h, #0
 ; CHECK-NEXT:    cmeq v3.8h, v1.8h, v3.8h
 ; CHECK-NEXT:    cmeq v1.8h, v1.8h, v16.8h
-; CHECK-NEXT:    bic v16.16b, v5.16b, v7.16b
-; CHECK-NEXT:    bic v7.8h, #128, lsl #8
-; CHECK-NEXT:    orr v7.16b, v7.16b, v16.16b
-; CHECK-NEXT:    cmlt v16.8h, v6.8h, #0
-; CHECK-NEXT:    bic v5.16b, v5.16b, v16.16b
-; CHECK-NEXT:    bic v16.8h, #128, lsl #8
 ; CHECK-NEXT:    mvn v2.16b, v2.16b
 ; CHECK-NEXT:    mvn v3.16b, v3.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    orr v5.16b, v16.16b, v5.16b
 ; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
 ; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bsl v0.16b, v7.16b, v4.16b
-; CHECK-NEXT:    bsl v1.16b, v5.16b, v6.16b
+; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
+; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %x, <16 x i16> %y)
   ret <16 x i16> %z
 }
 
 define <32 x i16> @v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {
 ; CHECK-LABEL: v32i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v18.8h, v1.8h, v5.8h
+; CHECK-NEXT:    sub v16.8h, v0.8h, v4.8h
+; CHECK-NEXT:    cmlt v24.8h, v16.8h, #0
+; CHECK-NEXT:    mvni v18.8h, #128, lsl #8
+; CHECK-NEXT:    sub v19.8h, v1.8h, v5.8h
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v18.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmlt v24.8h, v19.8h, #0
+; CHECK-NEXT:    mvni v20.8h, #128, lsl #8
+; CHECK-NEXT:    sub v21.8h, v2.8h, v6.8h
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmlt v24.8h, v21.8h, #0
+; CHECK-NEXT:    mvni v22.8h, #128, lsl #8
+; CHECK-NEXT:    sub v23.8h, v3.8h, v7.8h
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmlt v24.8h, v23.8h, #0
+; CHECK-NEXT:    mvni v17.8h, #128, lsl #8
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmge v4.8h, v4.8h, #0
+; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
+; CHECK-NEXT:    cmge v24.8h, v16.8h, #0
 ; CHECK-NEXT:    cmge v5.8h, v5.8h, #0
 ; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    sub v19.8h, v2.8h, v6.8h
-; CHECK-NEXT:    cmge v24.8h, v18.8h, #0
-; CHECK-NEXT:    sub v16.8h, v0.8h, v4.8h
+; CHECK-NEXT:    cmeq v4.8h, v0.8h, v4.8h
+; CHECK-NEXT:    cmeq v0.8h, v0.8h, v24.8h
+; CHECK-NEXT:    cmge v24.8h, v19.8h, #0
 ; CHECK-NEXT:    cmge v6.8h, v6.8h, #0
 ; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    sub v20.8h, v3.8h, v7.8h
 ; CHECK-NEXT:    cmeq v5.8h, v1.8h, v5.8h
 ; CHECK-NEXT:    cmeq v1.8h, v1.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v19.8h, #0
-; CHECK-NEXT:    movi v17.8h, #128, lsl #8
-; CHECK-NEXT:    cmge v4.8h, v4.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
+; CHECK-NEXT:    cmge v24.8h, v21.8h, #0
+; CHECK-NEXT:    mvn v4.16b, v4.16b
+; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    cmge v7.8h, v7.8h, #0
 ; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmlt v21.8h, v16.8h, #0
-; CHECK-NEXT:    cmge v22.8h, v16.8h, #0
 ; CHECK-NEXT:    cmeq v6.8h, v2.8h, v6.8h
 ; CHECK-NEXT:    cmeq v2.8h, v2.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v20.8h, #0
-; CHECK-NEXT:    cmeq v4.8h, v0.8h, v4.8h
-; CHECK-NEXT:    cmlt v23.8h, v18.8h, #0
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v22.8h
-; CHECK-NEXT:    cmeq v7.8h, v3.8h, v7.8h
-; CHECK-NEXT:    cmeq v3.8h, v3.8h, v24.8h
-; CHECK-NEXT:    bic v24.16b, v17.16b, v21.16b
-; CHECK-NEXT:    bic v21.8h, #128, lsl #8
-; CHECK-NEXT:    cmlt v22.8h, v19.8h, #0
-; CHECK-NEXT:    orr v21.16b, v21.16b, v24.16b
-; CHECK-NEXT:    bic v24.16b, v17.16b, v23.16b
-; CHECK-NEXT:    bic v23.8h, #128, lsl #8
-; CHECK-NEXT:    mvn v4.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    orr v23.16b, v23.16b, v24.16b
-; CHECK-NEXT:    bic v24.16b, v17.16b, v22.16b
-; CHECK-NEXT:    bic v22.8h, #128, lsl #8
+; CHECK-NEXT:    cmge v24.8h, v23.8h, #0
 ; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
 ; CHECK-NEXT:    mvn v4.16b, v5.16b
 ; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    orr v22.16b, v22.16b, v24.16b
-; CHECK-NEXT:    cmlt v24.8h, v20.8h, #0
+; CHECK-NEXT:    cmeq v7.8h, v3.8h, v7.8h
+; CHECK-NEXT:    cmeq v3.8h, v3.8h, v24.8h
 ; CHECK-NEXT:    and v1.16b, v4.16b, v1.16b
 ; CHECK-NEXT:    mvn v4.16b, v6.16b
 ; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    bic v17.16b, v17.16b, v24.16b
-; CHECK-NEXT:    bic v24.8h, #128, lsl #8
 ; CHECK-NEXT:    and v2.16b, v4.16b, v2.16b
 ; CHECK-NEXT:    mvn v4.16b, v7.16b
 ; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    orr v17.16b, v24.16b, v17.16b
 ; CHECK-NEXT:    and v3.16b, v4.16b, v3.16b
-; CHECK-NEXT:    bsl v0.16b, v21.16b, v16.16b
-; CHECK-NEXT:    bsl v1.16b, v23.16b, v18.16b
-; CHECK-NEXT:    bsl v2.16b, v22.16b, v19.16b
-; CHECK-NEXT:    bsl v3.16b, v17.16b, v20.16b
+; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
+; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
+; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
+; CHECK-NEXT:    bsl v3.16b, v17.16b, v23.16b
 ; CHECK-NEXT:    ret
   %z = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> %x, <32 x i16> %y)
   ret <32 x i16> %z
 }
 
 define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v8i8:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    mov v1.h[2], w8
 ; CHECK-NEXT:    mov v0.h[3], w9
 ; CHECK-NEXT:    mov v1.h[3], w10
 ; CHECK-NEXT:    shl v1.4h, v1.4h, #8
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #8
 ; CHECK-NEXT:    sub v3.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
 ; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v4.4h, v3.4h, #0
-; CHECK-NEXT:    movi v2.4h, #128, lsl #8
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v4.4h
+; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
 ; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    bic v2.8b, v2.8b, v4.8b
-; CHECK-NEXT:    bic v4.4h, #128, lsl #8
+; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
+; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
+; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
+; CHECK-NEXT:    mvn v5.8b, v4.8b
 ; CHECK-NEXT:    mvn v1.8b, v1.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    orr v2.8b, v4.8b, v2.8b
+; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
 ; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
 ; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    str s0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <4 x i8>, <4 x i8>* %px
   %y = load <4 x i8>, <4 x i8>* %py

 ; CHECK-NEXT:    fmov s2, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
 ; CHECK-NEXT:    mov v2.s[1], w11
 ; CHECK-NEXT:    shl v2.2s, v2.2s, #24
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #24
 ; CHECK-NEXT:    sub v3.2s, v0.2s, v2.2s
 ; CHECK-NEXT:    cmge v2.2s, v2.2s, #0
 ; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v4.2s, v3.2s, #0
-; CHECK-NEXT:    movi v1.2s, #128, lsl #24
-; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v4.2s
+; CHECK-NEXT:    cmge v5.2s, v3.2s, #0
 ; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    bic v1.8b, v1.8b, v4.8b
-; CHECK-NEXT:    bic v4.2s, #128, lsl #24
+; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
+; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
+; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
+; CHECK-NEXT:    mvn v5.8b, v4.8b
 ; CHECK-NEXT:    mvn v2.8b, v2.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    orr v1.8b, v4.8b, v1.8b
+; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
 ; CHECK-NEXT:    and v0.8b, v2.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #24
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
 ; CHECK-NEXT:    strb w8, [x2, #1]
 ; CHECK-NEXT:    strb w9, [x2]
 ; CHECK-NEXT:    ret
   %x = load <2 x i8>, <2 x i8>* %px
   %y = load <2 x i8>, <2 x i8>* %py
   %z = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %x, <2 x i8> %y)
   store <2 x i8> %z, <2 x i8>* %pz
   ret void
 }
 
 define void @v4i16(<4 x i16>* %px, <4 x i16>* %py, <4 x i16>* %pz) nounwind {
 ; CHECK-LABEL: v4i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    movi v2.4h, #128, lsl #8
+; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
 ; CHECK-NEXT:    sub v3.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
 ; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v4.4h
+; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
 ; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    bic v2.8b, v2.8b, v4.8b
-; CHECK-NEXT:    bic v4.4h, #128, lsl #8
+; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
+; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
+; CHECK-NEXT:    mvn v5.8b, v4.8b
 ; CHECK-NEXT:    mvn v1.8b, v1.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    orr v2.8b, v4.8b, v2.8b
+; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
 ; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    str d0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <4 x i16>, <4 x i16>* %px
   %y = load <4 x i16>, <4 x i16>* %py
   %z = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> %x, <4 x i16> %y)
   store <4 x i16> %z, <4 x i16>* %pz

 ; CHECK-NEXT:    fmov s2, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
 ; CHECK-NEXT:    mov v2.s[1], w11
 ; CHECK-NEXT:    shl v2.2s, v2.2s, #16
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #16
 ; CHECK-NEXT:    sub v3.2s, v0.2s, v2.2s
 ; CHECK-NEXT:    cmge v2.2s, v2.2s, #0
 ; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v4.2s, v3.2s, #0
-; CHECK-NEXT:    movi v1.2s, #128, lsl #24
-; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v4.2s
+; CHECK-NEXT:    cmge v5.2s, v3.2s, #0
 ; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    bic v1.8b, v1.8b, v4.8b
-; CHECK-NEXT:    bic v4.2s, #128, lsl #24
+; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
+; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
+; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
+; CHECK-NEXT:    mvn v5.8b, v4.8b
 ; CHECK-NEXT:    mvn v2.8b, v2.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    orr v1.8b, v4.8b, v1.8b
+; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
 ; CHECK-NEXT:    and v0.8b, v2.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #16
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
 ; CHECK-NEXT:    strh w8, [x2, #2]
 ; CHECK-NEXT:    strh w9, [x2]
 ; CHECK-NEXT:    ret

 ; CHECK-NEXT:    ret
   %z = call <12 x i8> @llvm.ssub.sat.v12i8(<12 x i8> %x, <12 x i8> %y)
   ret <12 x i8> %z
 }
 
 define void @v12i16(<12 x i16>* %px, <12 x i16>* %py, <12 x i16>* %pz) nounwind {
 ; CHECK-LABEL: v12i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q2, q0, [x0]
-; CHECK-NEXT:    ldp q3, q1, [x1]
-; CHECK-NEXT:    movi v4.8h, #128, lsl #8
-; CHECK-NEXT:    sub v5.8h, v0.8h, v1.8h
+; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q3, q2, [x1]
+; CHECK-NEXT:    mvni v5.8h, #128, lsl #8
+; CHECK-NEXT:    mvni v4.8h, #128, lsl #8
+; CHECK-NEXT:    sub v6.8h, v1.8h, v2.8h
+; CHECK-NEXT:    cmlt v16.8h, v6.8h, #0
+; CHECK-NEXT:    sub v7.8h, v0.8h, v3.8h
+; CHECK-NEXT:    mvn v17.16b, v16.16b
+; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmlt v16.8h, v7.8h, #0
+; CHECK-NEXT:    mvn v17.16b, v16.16b
+; CHECK-NEXT:    bsl v4.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
 ; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    sub v6.8h, v2.8h, v3.8h
-; CHECK-NEXT:    cmge v16.8h, v5.8h, #0
+; CHECK-NEXT:    cmge v16.8h, v6.8h, #0
 ; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    cmlt v7.8h, v5.8h, #0
-; CHECK-NEXT:    cmeq v1.8h, v0.8h, v1.8h
+; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
+; CHECK-NEXT:    cmeq v2.8h, v1.8h, v2.8h
+; CHECK-NEXT:    cmeq v1.8h, v1.8h, v16.8h
+; CHECK-NEXT:    cmge v16.8h, v7.8h, #0
+; CHECK-NEXT:    cmeq v3.8h, v0.8h, v3.8h
 ; CHECK-NEXT:    cmeq v0.8h, v0.8h, v16.8h
-; CHECK-NEXT:    cmge v16.8h, v6.8h, #0
-; CHECK-NEXT:    cmeq v3.8h, v2.8h, v3.8h
-; CHECK-NEXT:    cmeq v2.8h, v2.8h, v16.8h
-; CHECK-NEXT:    bic v16.16b, v4.16b, v7.16b
-; CHECK-NEXT:    bic v7.8h, #128, lsl #8
-; CHECK-NEXT:    orr v7.16b, v7.16b, v16.16b
-; CHECK-NEXT:    cmlt v16.8h, v6.8h, #0
-; CHECK-NEXT:    bic v4.16b, v4.16b, v16.16b
-; CHECK-NEXT:    bic v16.8h, #128, lsl #8
-; CHECK-NEXT:    mvn v1.16b, v1.16b
+; CHECK-NEXT:    mvn v2.16b, v2.16b
 ; CHECK-NEXT:    mvn v3.16b, v3.16b
+; CHECK-NEXT:    mvn v1.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    orr v4.16b, v16.16b, v4.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v2.16b
-; CHECK-NEXT:    bsl v0.16b, v7.16b, v5.16b
-; CHECK-NEXT:    bsl v1.16b, v4.16b, v6.16b
-; CHECK-NEXT:    str q1, [x2]
-; CHECK-NEXT:    str d0, [x2, #16]
+; CHECK-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-NEXT:    and v0.16b, v3.16b, v0.16b
+; CHECK-NEXT:    bsl v1.16b, v5.16b, v6.16b
+; CHECK-NEXT:    bsl v0.16b, v4.16b, v7.16b
+; CHECK-NEXT:    str q0, [x2]
+; CHECK-NEXT:    str d1, [x2, #16]
 ; CHECK-NEXT:    ret
   %x = load <12 x i16>, <12 x i16>* %px
   %y = load <12 x i16>, <12 x i16>* %py
   %z = call <12 x i16> @llvm.ssub.sat.v12i16(<12 x i16> %x, <12 x i16> %y)
   store <12 x i16> %z, <12 x i16>* %pz
   ret void
 }
 
; CHECK-NEXT:    ret
   ret void
 }
 
 define void @v1i16(<1 x i16>* %px, <1 x i16>* %py, <1 x i16>* %pz) nounwind {
 ; CHECK-LABEL: v1i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    ldr h1, [x1]
-; CHECK-NEXT:    movi v2.4h, #128, lsl #8
+; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
 ; CHECK-NEXT:    sub v3.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
 ; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v4.4h
+; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
 ; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    bic v2.8b, v2.8b, v4.8b
-; CHECK-NEXT:    bic v4.4h, #128, lsl #8
+; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
+; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
+; CHECK-NEXT:    mvn v5.8b, v4.8b
 ; CHECK-NEXT:    mvn v1.8b, v1.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    orr v2.8b, v4.8b, v2.8b
+; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
 ; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    str h0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <1 x i16>, <1 x i16>* %px
   %y = load <1 x i16>, <1 x i16>* %py
   %z = call <1 x i16> @llvm.ssub.sat.v1i16(<1 x i16> %x, <1 x i16> %y)
   store <1 x i16> %z, <1 x i16>* %pz

 }
 
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v2.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    cmge v1.2s, v1.2s, #0
 ; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v4.2s, v2.2s, #0
-; CHECK-NEXT:    movi v3.2s, #128, lsl #24
-; CHECK-NEXT:    cmeq v1.2s, v0.2s, v1.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v4.2s
+; CHECK-NEXT:    cmge v5.2s, v2.2s, #0
 ; CHECK-NEXT:    cmlt v4.2s, v2.2s, #0
-; CHECK-NEXT:    bic v3.8b, v3.8b, v4.8b
-; CHECK-NEXT:    bic v4.2s, #128, lsl #24
+; CHECK-NEXT:    cmeq v1.2s, v0.2s, v1.2s
+; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
+; CHECK-NEXT:    mvni v3.2s, #128, lsl #24
+; CHECK-NEXT:    mvn v5.8b, v4.8b
 ; CHECK-NEXT:    mvn v1.8b, v1.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    orr v3.8b, v4.8b, v3.8b
+; CHECK-NEXT:    bsl v3.8b, v4.8b, v5.8b
 ; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v3.8b, v2.8b
 ; CHECK-NEXT:    ret
   %z = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
   ret <2 x i32> %z
 }
 
 define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: v4i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v2.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
 ; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v4.4s, v2.4s, #0
-; CHECK-NEXT:    movi v3.4s, #128, lsl #24
-; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v4.4s
+; CHECK-NEXT:    cmge v5.4s, v2.4s, #0
 ; CHECK-NEXT:    cmlt v4.4s, v2.4s, #0
-; CHECK-NEXT:    bic v3.16b, v3.16b, v4.16b
-; CHECK-NEXT:    bic v4.4s, #128, lsl #24
+; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v5.4s
+; CHECK-NEXT:    mvni v3.4s, #128, lsl #24
+; CHECK-NEXT:    mvn v5.16b, v4.16b
 ; CHECK-NEXT:    mvn v1.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    orr v3.16b, v4.16b, v3.16b
+; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
 ; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %z = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
   ret <4 x i32> %z
 }
 
 define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; CHECK-LABEL: v8i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v4.4s, v0.4s, v2.4s
+; CHECK-NEXT:    cmlt v16.4s, v4.4s, #0
+; CHECK-NEXT:    mvni v6.4s, #128, lsl #24
+; CHECK-NEXT:    sub v7.4s, v1.4s, v3.4s
+; CHECK-NEXT:    mvn v17.16b, v16.16b
+; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmlt v16.4s, v7.4s, #0
+; CHECK-NEXT:    mvni v5.4s, #128, lsl #24
+; CHECK-NEXT:    mvn v17.16b, v16.16b
+; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
 ; CHECK-NEXT:    cmge v2.4s, v2.4s, #0
 ; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    sub v6.4s, v1.4s, v3.4s
 ; CHECK-NEXT:    cmge v16.4s, v4.4s, #0
-; CHECK-NEXT:    movi v5.4s, #128, lsl #24
 ; CHECK-NEXT:    cmge v3.4s, v3.4s, #0
 ; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmlt v7.4s, v4.4s, #0
 ; CHECK-NEXT:    cmeq v2.4s, v0.4s, v2.4s
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, v16.4s
-; CHECK-NEXT:    cmge v16.4s, v6.4s, #0
+; CHECK-NEXT:    cmge v16.4s, v7.4s, #0
 ; CHECK-NEXT:    cmeq v3.4s, v1.4s, v3.4s
 ; CHECK-NEXT:    cmeq v1.4s, v1.4s, v16.4s
-; CHECK-NEXT:    bic v16.16b, v5.16b, v7.16b
-; CHECK-NEXT:    bic v7.4s, #128, lsl #24
-; CHECK-NEXT:    orr v7.16b, v7.16b, v16.16b
-; CHECK-NEXT:    cmlt v16.4s, v6.4s, #0
-; CHECK-NEXT:    bic v5.16b, v5.16b, v16.16b
-; CHECK-NEXT:    bic v16.4s, #128, lsl #24
 ; CHECK-NEXT:    mvn v2.16b, v2.16b
 ; CHECK-NEXT:    mvn v3.16b, v3.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    orr v5.16b, v16.16b, v5.16b
 ; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
 ; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bsl v0.16b, v7.16b, v4.16b
-; CHECK-NEXT:    bsl v1.16b, v5.16b, v6.16b
+; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
+; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
 ; CHECK-NEXT:    ret
   %z = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
   ret <8 x i32> %z
 }
 
 define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 ; CHECK-LABEL: v16i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub v18.4s, v1.4s, v5.4s
+; CHECK-NEXT:    sub v16.4s, v0.4s, v4.4s
+; CHECK-NEXT:    cmlt v24.4s, v16.4s, #0
+; CHECK-NEXT:    mvni v18.4s, #128, lsl #24
+; CHECK-NEXT:    sub v19.4s, v1.4s, v5.4s
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v18.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmlt v24.4s, v19.4s, #0
+; CHECK-NEXT:    mvni v20.4s, #128, lsl #24
+; CHECK-NEXT:    sub v21.4s, v2.4s, v6.4s
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmlt v24.4s, v21.4s, #0
+; CHECK-NEXT:    mvni v22.4s, #128, lsl #24
+; CHECK-NEXT:    sub v23.4s, v3.4s, v7.4s
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmlt v24.4s, v23.4s, #0
+; CHECK-NEXT:    mvni v17.4s, #128, lsl #24
+; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
+; CHECK-NEXT:    cmge v4.4s, v4.4s, #0
+; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
+; CHECK-NEXT:    cmge v24.4s, v16.4s, #0
 ; CHECK-NEXT:    cmge v5.4s, v5.4s, #0
 ; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    sub v19.4s, v2.4s, v6.4s
-; CHECK-NEXT:    cmge v24.4s, v18.4s, #0
-; CHECK-NEXT:    sub v16.4s, v0.4s, v4.4s
+; CHECK-NEXT:    cmeq v4.4s, v0.4s, v4.4s
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v24.4s
+; CHECK-NEXT:    cmge v24.4s, v19.4s, #0
 ; CHECK-NEXT:    cmge v6.4s, v6.4s, #0
 ; CHECK-NEXT:    cmge v2.4s, v2.4s, #0
-; CHECK-NEXT:    sub v20.4s, v3.4s, v7.4s
 ; CHECK-NEXT:    cmeq v5.4s, v1.4s, v5.4s
 ; CHECK-NEXT:    cmeq v1.4s, v1.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v19.4s, #0
-; CHECK-NEXT:    movi v17.4s, #128, lsl #24
-; CHECK-NEXT:    cmge v4.4s, v4.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
+; CHECK-NEXT:    cmge v24.4s, v21.4s, #0
+; CHECK-NEXT:    mvn v4.16b, v4.16b
+; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    cmge v7.4s, v7.4s, #0
 ; CHECK-NEXT:    cmge v3.4s, v3.4s, #0
-; CHECK-NEXT:    cmlt v21.4s, v16.4s, #0
-; CHECK-NEXT:    cmge v22.4s, v16.4s, #0
 ; CHECK-NEXT:    cmeq v6.4s, v2.4s, v6.4s
 ; CHECK-NEXT:    cmeq v2.4s, v2.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v20.4s, #0
-; CHECK-NEXT:    cmeq v4.4s, v0.4s, v4.4s
-; CHECK-NEXT:    cmlt v23.4s, v18.4s, #0
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v22.4s
-; CHECK-NEXT:    cmeq v7.4s, v3.4s, v7.4s
-; CHECK-NEXT:    cmeq v3.4s, v3.4s, v24.4s
-; CHECK-NEXT:    bic v24.16b, v17.16b, v21.16b
-; CHECK-NEXT:    bic v21.4s, #128, lsl #24
-; CHECK-NEXT:    cmlt v22.4s, v19.4s, #0
-; CHECK-NEXT:    orr v21.16b, v21.16b, v24.16b
-; CHECK-NEXT:    bic v24.16b, v17.16b, v23.16b
-; CHECK-NEXT:    bic v23.4s, #128, lsl #24
-; CHECK-NEXT:    mvn v4.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    orr v23.16b, v23.16b, v24.16b
-; CHECK-NEXT:    bic v24.16b, v17.16b, v22.16b
-; CHECK-NEXT:    bic v22.4s, #128, lsl #24
+; CHECK-NEXT:    cmge v24.4s, v23.4s, #0
 ; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
 ; CHECK-NEXT:    mvn v4.16b, v5.16b
 ; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    orr v22.16b, v22.16b, v24.16b
-; CHECK-NEXT:    cmlt v24.4s, v20.4s, #0
+; CHECK-NEXT:    cmeq v7.4s, v3.4s, v7.4s
+; CHECK-NEXT:    cmeq v3.4s, v3.4s, v24.4s
 ; CHECK-NEXT:    and v1.16b, v4.16b, v1.16b
 ; CHECK-NEXT:    mvn v4.16b, v6.16b
 ; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    bic v17.16b, v17.16b, v24.16b
-; CHECK-NEXT:    bic v24.4s, #128, lsl #24
 ; CHECK-NEXT:    and v2.16b, v4.16b, v2.16b
 ; CHECK-NEXT:    mvn v4.16b, v7.16b
 ; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    orr v17.16b, v24.16b, v17.16b
 ; CHECK-NEXT:    and v3.16b, v4.16b, v3.16b
-; CHECK-NEXT:    bsl v0.16b, v21.16b, v16.16b
-; CHECK-NEXT:    bsl v1.16b, v23.16b, v18.16b
-; CHECK-NEXT:    bsl v2.16b, v22.16b, v19.16b
-; CHECK-NEXT:    bsl v3.16b, v17.16b, v20.16b
+; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
+; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
+; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
+; CHECK-NEXT:    bsl v3.16b, v17.16b, v23.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
   ret <16 x i32> %z
 }
 
 define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: v2i64:
 ; CHECK:       // %bb.0:

llvm/trunk/test/CodeGen/AArch64/uadd_sat_vec.ll

 }
 
 define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: v2i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v1.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    cmhi v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
   ret <2 x i64> %z
 }
 
 define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; CHECK-LABEL: v4i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v2.2d, v0.2d, v2.2d
 ; CHECK-NEXT:    add v3.2d, v1.2d, v3.2d
 ; CHECK-NEXT:    cmhi v0.2d, v0.2d, v2.2d
 ; CHECK-NEXT:    cmhi v1.2d, v1.2d, v3.2d
 ; CHECK-NEXT:    bic v2.16b, v2.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    bic v3.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bic v1.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    orr v1.16b, v1.16b, v3.16b
 ; CHECK-NEXT:    ret
   %z = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
   ret <4 x i64> %z
 }
 
 define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; CHECK-LABEL: v8i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v4.2d, v0.2d, v4.2d
 ; CHECK-NEXT:    add v5.2d, v1.2d, v5.2d
 ; CHECK-NEXT:    add v6.2d, v2.2d, v6.2d
 ; CHECK-NEXT:    add v7.2d, v3.2d, v7.2d
 ; CHECK-NEXT:    cmhi v0.2d, v0.2d, v4.2d
 ; CHECK-NEXT:    cmhi v1.2d, v1.2d, v5.2d
 ; CHECK-NEXT:    cmhi v2.2d, v2.2d, v6.2d
 ; CHECK-NEXT:    cmhi v3.2d, v3.2d, v7.2d
 ; CHECK-NEXT:    bic v4.16b, v4.16b, v0.16b
-; CHECK-NEXT:    bic v0.4s, #0
 ; CHECK-NEXT:    bic v5.16b, v5.16b, v1.16b
-; CHECK-NEXT:    bic v1.4s, #0
 ; CHECK-NEXT:    bic v6.16b, v6.16b, v2.16b
-; CHECK-NEXT:    bic v2.4s, #0
 ; CHECK-NEXT:    bic v7.16b, v7.16b, v3.16b
-; CHECK-NEXT:    bic v3.4s, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v4.16b
 ; CHECK-NEXT:    orr v1.16b, v1.16b, v5.16b
 ; CHECK-NEXT:    orr v2.16b, v2.16b, v6.16b
 ; CHECK-NEXT:    orr v3.16b, v3.16b, v7.16b
 ; CHECK-NEXT:    ret
   %z = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
   ret <8 x i64> %z
 }