[Codegen] Alter the default promotion for add_sat and sub_sat

[dmgreen]

Summary

The default promotion for the add_sat/sub_sat nodes currently does:

//   1. ANY_EXTEND iN to iM
//   2. SHL by M-N
//   3. [US][ADD|SUB]SAT
//   4. L/ASHR by M-N

If the promoted add_sat or sub_sat node is not legal, this can produce code that effectively does a lot of shifting (and requiring large constants to be materialised) just to use the overflow flag. It is simpler to just do the saturation manually, using the higher bitwidth addition and a min/max against the saturating bounds. That is what this patch attempts to do.

A few points of interest:

• This still uses the existing promotion when the promoted add/sub_sat is legal. In many situations (but not all) it would probably be better to just perform the new promotion.
• It always creates a MIN and MAX node, even if they are not legal. The alternative would be to be to create a cmp/select pair, which these will legalise into anyway.
• We only have AArch64 and X86 tests. I have added ARM (for which just the differences are shown here). I'm happy to add other architectures if people are interested.

[nikic]

LGTM

This still uses the existing promotion when the promoted add/sub_sat is legal. In many situations (but not all) it would probably be better to just perform the new promotion.

I guess at least for the unsigned case, it would be reasonable to use the shift+sat expansion only if both the sat is legal *and* min/max is not legal. The umax+sub expansion at least is clearly better than usubsat + 3 shifts.

But that's probably better left for a followup patch.

[dmgreen]

Thanks. I'll put a patch together showing the differences. It was only really the odd types like "i4" that changed IIRC. Some of them were looking better in places, worse in others. Making it dependent on whether min/max are available sounds like a good idea.

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
   // the new one.
   ReplaceValueWith(SDValue(N, 0), Res);
 
   // Convert to the expected type.
   return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) {
-  // For promoting iN -> iM, this can be expanded by
-  // 1. ANY_EXTEND iN to iM
-  // 2. SHL by M-N
-  // 3. [US][ADD|SUB]SAT
-  // 4. L/ASHR by M-N
+  // If the promoted type is legal, we can convert this to:
+  //   1. ANY_EXTEND iN to iM
+  //   2. SHL by M-N
+  //   3. [US][ADD|SUB]SAT
+  //   4. L/ASHR by M-N
+  // Else it is more efficient to convert this to a min and a max
+  // operation in the higher precision arithmetic.
   SDLoc dl(N);
   SDValue Op1 = N->getOperand(0);
   SDValue Op2 = N->getOperand(1);
   unsigned OldBits = Op1.getScalarValueSizeInBits();
 
   unsigned Opcode = N->getOpcode();
+
+  SDValue Op1Promoted = GetPromotedInteger(Op1);
+  SDValue Op2Promoted = GetPromotedInteger(Op2);
+  EVT PromotedType = Op1Promoted.getValueType();
+  unsigned NewBits = PromotedType.getScalarSizeInBits();
+
+  if (TLI.isOperationLegalOrCustom(Opcode, PromotedType)) {
     unsigned ShiftOp;
     switch (Opcode) {
     case ISD::SADDSAT:
     case ISD::SSUBSAT:
       ShiftOp = ISD::SRA;
       break;
     case ISD::UADDSAT:
     case ISD::USUBSAT:
       ShiftOp = ISD::SRL;
       break;
     default:
       llvm_unreachable("Expected opcode to be signed or unsigned saturation "
                        "addition or subtraction");
     }
 
-  SDValue Op1Promoted = GetPromotedInteger(Op1);
-  SDValue Op2Promoted = GetPromotedInteger(Op2);
-
-  EVT PromotedType = Op1Promoted.getValueType();
-  unsigned NewBits = PromotedType.getScalarSizeInBits();
     unsigned SHLAmount = NewBits - OldBits;
     EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
     SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
     Op1Promoted =
         DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
     Op2Promoted =
         DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
 
     SDValue Result =
         DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
     return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
+  } else {
+    if (Opcode == ISD::USUBSAT) {
+      SDValue Max =
+          DAG.getNode(ISD::UMAX, dl, PromotedType, Op1Promoted, Op2Promoted);
+      return DAG.getNode(ISD::SUB, dl, PromotedType, Max, Op2Promoted);
+    }
+
+    if (Opcode == ISD::UADDSAT) {
+      APInt MaxVal = APInt::getAllOnesValue(OldBits).zext(NewBits);
+      SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
+      SDValue Add =
+          DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
+      return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
+    }
+
+    unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
+    APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
+    APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
+    SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
+    SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
+    SDValue Result =
+        DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
+    Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
+    Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
+    return Result;
+  }
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
   // Can just promote the operands then continue with operation.
   SDLoc dl(N);
   SDValue Op1Promoted, Op2Promoted;
   bool Signed =
       N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;

llvm/test/CodeGen/AArch64/sadd_sat.ll

 ; CHECK-NEXT:    ret
   %tmp = call i64 @llvm.sadd.sat.i64(i64 %x, i64 %y);
   ret i64 %tmp;
 }
 
 define i16 @func16(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: func16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #16
-; CHECK-NEXT:    adds w10, w8, w1, lsl #16
-; CHECK-NEXT:    mov w9, #2147483647
-; CHECK-NEXT:    cmp w10, #0 // =0
-; CHECK-NEXT:    cinv w9, w9, ge
-; CHECK-NEXT:    adds w8, w8, w1, lsl #16
-; CHECK-NEXT:    csel w8, w9, w8, vs
-; CHECK-NEXT:    asr w0, w8, #16
+; CHECK-NEXT:    add w8, w0, w1
+; CHECK-NEXT:    mov w9, #32767
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    csel w8, w8, w9, lt
+; CHECK-NEXT:    cmn w8, #8, lsl #12 // =32768
+; CHECK-NEXT:    mov w9, #-32768
+; CHECK-NEXT:    csel w0, w8, w9, gt
 ; CHECK-NEXT:    ret
   %tmp = call i16 @llvm.sadd.sat.i16(i16 %x, i16 %y);
   ret i16 %tmp;
 }
 
 define i8 @func8(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: func8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #24
-; CHECK-NEXT:    adds w10, w8, w1, lsl #24
-; CHECK-NEXT:    mov w9, #2147483647
-; CHECK-NEXT:    cmp w10, #0 // =0
-; CHECK-NEXT:    cinv w9, w9, ge
-; CHECK-NEXT:    adds w8, w8, w1, lsl #24
-; CHECK-NEXT:    csel w8, w9, w8, vs
-; CHECK-NEXT:    asr w0, w8, #24
+; CHECK-NEXT:    add w8, w0, w1
+; CHECK-NEXT:    mov w9, #127
+; CHECK-NEXT:    cmp w8, #127 // =127
+; CHECK-NEXT:    csel w8, w8, w9, lt
+; CHECK-NEXT:    cmn w8, #128 // =128
+; CHECK-NEXT:    mov w9, #-128
+; CHECK-NEXT:    csel w0, w8, w9, gt
 ; CHECK-NEXT:    ret
   %tmp = call i8 @llvm.sadd.sat.i8(i8 %x, i8 %y);
   ret i8 %tmp;
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; CHECK-LABEL: func3:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #28
-; CHECK-NEXT:    adds w10, w8, w1, lsl #28
-; CHECK-NEXT:    mov w9, #2147483647
-; CHECK-NEXT:    cmp w10, #0 // =0
-; CHECK-NEXT:    cinv w9, w9, ge
-; CHECK-NEXT:    adds w8, w8, w1, lsl #28
-; CHECK-NEXT:    csel w8, w9, w8, vs
-; CHECK-NEXT:    asr w0, w8, #28
+; CHECK-NEXT:    add w8, w0, w1
+; CHECK-NEXT:    mov w9, #7
+; CHECK-NEXT:    cmp w8, #7 // =7
+; CHECK-NEXT:    csel w8, w8, w9, lt
+; CHECK-NEXT:    cmn w8, #8 // =8
+; CHECK-NEXT:    mov w9, #-8
+; CHECK-NEXT:    csel w0, w8, w9, gt
 ; CHECK-NEXT:    ret
   %tmp = call i4 @llvm.sadd.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: vec:
 ; CHECK:       // %bb.0:

llvm/test/CodeGen/AArch64/sadd_sat_vec.ll

 
 define void @v4i8(<4 x i8>* %px, <4 x i8>* %py, <4 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldrb w8, [x0]
 ; CHECK-NEXT:    ldrb w9, [x1]
 ; CHECK-NEXT:    ldrb w10, [x0, #1]
 ; CHECK-NEXT:    ldrb w11, [x1, #1]
-; CHECK-NEXT:    ldrb w12, [x0, #2]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    ldrb w8, [x1, #2]
 ; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    ldrb w8, [x0, #2]
+; CHECK-NEXT:    ldrb w9, [x1, #2]
 ; CHECK-NEXT:    mov v0.h[1], w10
-; CHECK-NEXT:    ldrb w9, [x0, #3]
-; CHECK-NEXT:    ldrb w10, [x1, #3]
 ; CHECK-NEXT:    mov v1.h[1], w11
-; CHECK-NEXT:    mov v0.h[2], w12
-; CHECK-NEXT:    mov v1.h[2], w8
-; CHECK-NEXT:    mov v0.h[3], w9
-; CHECK-NEXT:    mov v1.h[3], w10
-; CHECK-NEXT:    shl v1.4h, v1.4h, #8
-; CHECK-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-NEXT:    add v3.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
-; CHECK-NEXT:    cmlt v1.4h, v1.4h, #0
-; CHECK-NEXT:    cmgt v0.4h, v0.4h, v3.4h
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    eor v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
-; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NEXT:    ldrb w10, [x0, #3]
+; CHECK-NEXT:    ldrb w11, [x1, #3]
+; CHECK-NEXT:    mov v0.h[2], w8
+; CHECK-NEXT:    mov v1.h[2], w9
+; CHECK-NEXT:    mov v0.h[3], w10
+; CHECK-NEXT:    mov v1.h[3], w11
+; CHECK-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    movi v1.4h, #127
+; CHECK-NEXT:    smin v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    mvni v1.4h, #127
+; CHECK-NEXT:    smax v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    str s0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <4 x i8>, <4 x i8>* %px
   %y = load <4 x i8>, <4 x i8>* %py
   %z = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> %x, <4 x i8> %y)
   store <4 x i8> %z, <4 x i8>* %pz
   ret void
 }
 
 define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v2i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldrb w8, [x0]
 ; CHECK-NEXT:    ldrb w9, [x1]
 ; CHECK-NEXT:    ldrb w10, [x0, #1]
 ; CHECK-NEXT:    ldrb w11, [x1, #1]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    fmov s2, w9
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
-; CHECK-NEXT:    mov v2.s[1], w11
-; CHECK-NEXT:    shl v2.2s, v2.2s, #24
-; CHECK-NEXT:    shl v0.2s, v0.2s, #24
-; CHECK-NEXT:    add v3.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
-; CHECK-NEXT:    cmlt v2.2s, v2.2s, #0
-; CHECK-NEXT:    cmgt v0.2s, v0.2s, v3.2s
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    eor v0.8b, v2.8b, v0.8b
-; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
-; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
-; CHECK-NEXT:    ushr v0.2s, v0.2s, #24
+; CHECK-NEXT:    mov v1.s[1], w11
+; CHECK-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    movi v1.2s, #127
+; CHECK-NEXT:    smin v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    mvni v1.2s, #127
+; CHECK-NEXT:    smax v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
 ; CHECK-NEXT:    strb w8, [x2, #1]
 ; CHECK-NEXT:    strb w9, [x2]
 ; CHECK-NEXT:    ret
   %x = load <2 x i8>, <2 x i8>* %px
   %y = load <2 x i8>, <2 x i8>* %py
   %z = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %x, <2 x i8> %y)

 define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 ; CHECK-LABEL: v2i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldrh w8, [x0]
 ; CHECK-NEXT:    ldrh w9, [x1]
 ; CHECK-NEXT:    ldrh w10, [x0, #2]
 ; CHECK-NEXT:    ldrh w11, [x1, #2]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    fmov s2, w9
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
-; CHECK-NEXT:    mov v2.s[1], w11
-; CHECK-NEXT:    shl v2.2s, v2.2s, #16
-; CHECK-NEXT:    shl v0.2s, v0.2s, #16
-; CHECK-NEXT:    add v3.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
-; CHECK-NEXT:    cmlt v2.2s, v2.2s, #0
-; CHECK-NEXT:    cmgt v0.2s, v0.2s, v3.2s
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    eor v0.8b, v2.8b, v0.8b
-; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
-; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
-; CHECK-NEXT:    ushr v0.2s, v0.2s, #16
+; CHECK-NEXT:    mov v1.s[1], w11
+; CHECK-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    movi v1.2s, #127, msl #8
+; CHECK-NEXT:    smin v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    mvni v1.2s, #127, msl #8
+; CHECK-NEXT:    smax v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
 ; CHECK-NEXT:    strh w8, [x2, #2]
 ; CHECK-NEXT:    strh w9, [x2]
 ; CHECK-NEXT:    ret
   %x = load <2 x i16>, <2 x i16>* %px
   %y = load <2 x i16>, <2 x i16>* %py
   %z = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %x, <2 x i16> %y)
; CHECK-NEXT:    ret
   %z = call <1 x i16> @llvm.sadd.sat.v1i16(<1 x i16> %x, <1 x i16> %y)
   store <1 x i16> %z, <1 x i16>* %pz
   ret void
 }
 
 define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; CHECK-LABEL: v16i4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    shl v1.16b, v1.16b, #4
-; CHECK-NEXT:    shl v0.16b, v0.16b, #4
-; CHECK-NEXT:    add v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmlt v4.16b, v3.16b, #0
-; CHECK-NEXT:    movi v2.16b, #127
-; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmgt v0.16b, v0.16b, v3.16b
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    bsl v2.16b, v4.16b, v5.16b
-; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v2.16b, v3.16b
-; CHECK-NEXT:    sshr v0.16b, v0.16b, #4
+; CHECK-NEXT:    add v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    movi v1.16b, #7
+; CHECK-NEXT:    smin v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    movi v1.16b, #248
+; CHECK-NEXT:    smax v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i4> @llvm.sadd.sat.v16i4(<16 x i4> %x, <16 x i4> %y)
   ret <16 x i4> %z
 }
 
 define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 ; CHECK-LABEL: v16i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    shl v1.16b, v1.16b, #7
-; CHECK-NEXT:    shl v0.16b, v0.16b, #7
-; CHECK-NEXT:    add v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmlt v4.16b, v3.16b, #0
-; CHECK-NEXT:    movi v2.16b, #127
-; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmgt v0.16b, v0.16b, v3.16b
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    bsl v2.16b, v4.16b, v5.16b
-; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v2.16b, v3.16b
-; CHECK-NEXT:    sshr v0.16b, v0.16b, #7
+; CHECK-NEXT:    add v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    smin v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    movi v1.2d, #0xffffffffffffffff
+; CHECK-NEXT:    smax v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i1> @llvm.sadd.sat.v16i1(<16 x i1> %x, <16 x i1> %y)
   ret <16 x i1> %z
 }
 
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:

llvm/test/CodeGen/AArch64/ssub_sat.ll

 ; CHECK-NEXT:    ret
   %tmp = call i64 @llvm.ssub.sat.i64(i64 %x, i64 %y);
   ret i64 %tmp;
 }
 
 define i16 @func16(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: func16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #16
-; CHECK-NEXT:    subs w10, w8, w1, lsl #16
-; CHECK-NEXT:    mov w9, #2147483647
-; CHECK-NEXT:    cmp w10, #0 // =0
-; CHECK-NEXT:    cinv w9, w9, ge
-; CHECK-NEXT:    subs w8, w8, w1, lsl #16
-; CHECK-NEXT:    csel w8, w9, w8, vs
-; CHECK-NEXT:    asr w0, w8, #16
+; CHECK-NEXT:    sub w8, w0, w1
+; CHECK-NEXT:    mov w9, #32767
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    csel w8, w8, w9, lt
+; CHECK-NEXT:    cmn w8, #8, lsl #12 // =32768
+; CHECK-NEXT:    mov w9, #-32768
+; CHECK-NEXT:    csel w0, w8, w9, gt
 ; CHECK-NEXT:    ret
   %tmp = call i16 @llvm.ssub.sat.i16(i16 %x, i16 %y);
   ret i16 %tmp;
 }
 
 define i8 @func8(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: func8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #24
-; CHECK-NEXT:    subs w10, w8, w1, lsl #24
-; CHECK-NEXT:    mov w9, #2147483647
-; CHECK-NEXT:    cmp w10, #0 // =0
-; CHECK-NEXT:    cinv w9, w9, ge
-; CHECK-NEXT:    subs w8, w8, w1, lsl #24
-; CHECK-NEXT:    csel w8, w9, w8, vs
-; CHECK-NEXT:    asr w0, w8, #24
+; CHECK-NEXT:    sub w8, w0, w1
+; CHECK-NEXT:    mov w9, #127
+; CHECK-NEXT:    cmp w8, #127 // =127
+; CHECK-NEXT:    csel w8, w8, w9, lt
+; CHECK-NEXT:    cmn w8, #128 // =128
+; CHECK-NEXT:    mov w9, #-128
+; CHECK-NEXT:    csel w0, w8, w9, gt
 ; CHECK-NEXT:    ret
   %tmp = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y);
   ret i8 %tmp;
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; CHECK-LABEL: func3:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #28
-; CHECK-NEXT:    subs w10, w8, w1, lsl #28
-; CHECK-NEXT:    mov w9, #2147483647
-; CHECK-NEXT:    cmp w10, #0 // =0
-; CHECK-NEXT:    cinv w9, w9, ge
-; CHECK-NEXT:    subs w8, w8, w1, lsl #28
-; CHECK-NEXT:    csel w8, w9, w8, vs
-; CHECK-NEXT:    asr w0, w8, #28
+; CHECK-NEXT:    sub w8, w0, w1
+; CHECK-NEXT:    mov w9, #7
+; CHECK-NEXT:    cmp w8, #7 // =7
+; CHECK-NEXT:    csel w8, w8, w9, lt
+; CHECK-NEXT:    cmn w8, #8 // =8
+; CHECK-NEXT:    mov w9, #-8
+; CHECK-NEXT:    csel w0, w8, w9, gt
 ; CHECK-NEXT:    ret
   %tmp = call i4 @llvm.ssub.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: vec:
 ; CHECK:       // %bb.0:

llvm/test/CodeGen/AArch64/ssub_sat_vec.ll

 
 define void @v4i8(<4 x i8>* %px, <4 x i8>* %py, <4 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldrb w8, [x0]
 ; CHECK-NEXT:    ldrb w9, [x1]
 ; CHECK-NEXT:    ldrb w10, [x0, #1]
 ; CHECK-NEXT:    ldrb w11, [x1, #1]
-; CHECK-NEXT:    ldrb w12, [x0, #2]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    ldrb w8, [x1, #2]
 ; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    ldrb w8, [x0, #2]
+; CHECK-NEXT:    ldrb w9, [x1, #2]
 ; CHECK-NEXT:    mov v0.h[1], w10
-; CHECK-NEXT:    ldrb w9, [x0, #3]
-; CHECK-NEXT:    ldrb w10, [x1, #3]
 ; CHECK-NEXT:    mov v1.h[1], w11
-; CHECK-NEXT:    mov v0.h[2], w12
-; CHECK-NEXT:    mov v1.h[2], w8
-; CHECK-NEXT:    mov v0.h[3], w9
-; CHECK-NEXT:    mov v1.h[3], w10
-; CHECK-NEXT:    shl v1.4h, v1.4h, #8
-; CHECK-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-NEXT:    sub v3.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
-; CHECK-NEXT:    cmgt v1.4h, v1.4h, #0
-; CHECK-NEXT:    cmgt v0.4h, v0.4h, v3.4h
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    eor v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
-; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NEXT:    ldrb w10, [x0, #3]
+; CHECK-NEXT:    ldrb w11, [x1, #3]
+; CHECK-NEXT:    mov v0.h[2], w8
+; CHECK-NEXT:    mov v1.h[2], w9
+; CHECK-NEXT:    mov v0.h[3], w10
+; CHECK-NEXT:    mov v1.h[3], w11
+; CHECK-NEXT:    sub v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    movi v1.4h, #127
+; CHECK-NEXT:    smin v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    mvni v1.4h, #127
+; CHECK-NEXT:    smax v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    str s0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <4 x i8>, <4 x i8>* %px
   %y = load <4 x i8>, <4 x i8>* %py
   %z = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> %x, <4 x i8> %y)
   store <4 x i8> %z, <4 x i8>* %pz
   ret void
 }
 
 define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v2i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldrb w8, [x0]
 ; CHECK-NEXT:    ldrb w9, [x1]
 ; CHECK-NEXT:    ldrb w10, [x0, #1]
 ; CHECK-NEXT:    ldrb w11, [x1, #1]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    fmov s2, w9
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
-; CHECK-NEXT:    mov v2.s[1], w11
-; CHECK-NEXT:    shl v2.2s, v2.2s, #24
-; CHECK-NEXT:    shl v0.2s, v0.2s, #24
-; CHECK-NEXT:    sub v3.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
-; CHECK-NEXT:    cmgt v2.2s, v2.2s, #0
-; CHECK-NEXT:    cmgt v0.2s, v0.2s, v3.2s
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    eor v0.8b, v2.8b, v0.8b
-; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
-; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
-; CHECK-NEXT:    ushr v0.2s, v0.2s, #24
+; CHECK-NEXT:    mov v1.s[1], w11
+; CHECK-NEXT:    sub v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    movi v1.2s, #127
+; CHECK-NEXT:    smin v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    mvni v1.2s, #127
+; CHECK-NEXT:    smax v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
 ; CHECK-NEXT:    strb w8, [x2, #1]
 ; CHECK-NEXT:    strb w9, [x2]
 ; CHECK-NEXT:    ret
   %x = load <2 x i8>, <2 x i8>* %px
   %y = load <2 x i8>, <2 x i8>* %py
   %z = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %x, <2 x i8> %y)

 define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 ; CHECK-LABEL: v2i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldrh w8, [x0]
 ; CHECK-NEXT:    ldrh w9, [x1]
 ; CHECK-NEXT:    ldrh w10, [x0, #2]
 ; CHECK-NEXT:    ldrh w11, [x1, #2]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    fmov s2, w9
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
-; CHECK-NEXT:    mov v2.s[1], w11
-; CHECK-NEXT:    shl v2.2s, v2.2s, #16
-; CHECK-NEXT:    shl v0.2s, v0.2s, #16
-; CHECK-NEXT:    sub v3.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
-; CHECK-NEXT:    cmgt v2.2s, v2.2s, #0
-; CHECK-NEXT:    cmgt v0.2s, v0.2s, v3.2s
-; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    eor v0.8b, v2.8b, v0.8b
-; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
-; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
-; CHECK-NEXT:    ushr v0.2s, v0.2s, #16
+; CHECK-NEXT:    mov v1.s[1], w11
+; CHECK-NEXT:    sub v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    movi v1.2s, #127, msl #8
+; CHECK-NEXT:    smin v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    mvni v1.2s, #127, msl #8
+; CHECK-NEXT:    smax v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
 ; CHECK-NEXT:    strh w8, [x2, #2]
 ; CHECK-NEXT:    strh w9, [x2]
 ; CHECK-NEXT:    ret
   %x = load <2 x i16>, <2 x i16>* %px
   %y = load <2 x i16>, <2 x i16>* %py
   %z = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %x, <2 x i16> %y)
; CHECK-NEXT:    ret
   %z = call <1 x i16> @llvm.ssub.sat.v1i16(<1 x i16> %x, <1 x i16> %y)
   store <1 x i16> %z, <1 x i16>* %pz
   ret void
 }
 
 define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; CHECK-LABEL: v16i4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    shl v1.16b, v1.16b, #4
-; CHECK-NEXT:    shl v0.16b, v0.16b, #4
-; CHECK-NEXT:    sub v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmlt v4.16b, v3.16b, #0
-; CHECK-NEXT:    movi v2.16b, #127
-; CHECK-NEXT:    cmgt v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmgt v0.16b, v0.16b, v3.16b
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    bsl v2.16b, v4.16b, v5.16b
-; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v2.16b, v3.16b
-; CHECK-NEXT:    sshr v0.16b, v0.16b, #4
+; CHECK-NEXT:    sub v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    movi v1.16b, #7
+; CHECK-NEXT:    smin v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    movi v1.16b, #248
+; CHECK-NEXT:    smax v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i4> @llvm.ssub.sat.v16i4(<16 x i4> %x, <16 x i4> %y)
   ret <16 x i4> %z
 }
 
 define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 ; CHECK-LABEL: v16i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    shl v1.16b, v1.16b, #7
-; CHECK-NEXT:    shl v0.16b, v0.16b, #7
-; CHECK-NEXT:    sub v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmlt v4.16b, v3.16b, #0
-; CHECK-NEXT:    movi v2.16b, #127
-; CHECK-NEXT:    cmgt v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmgt v0.16b, v0.16b, v3.16b
-; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    bsl v2.16b, v4.16b, v5.16b
-; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    bsl v0.16b, v2.16b, v3.16b
-; CHECK-NEXT:    sshr v0.16b, v0.16b, #7
+; CHECK-NEXT:    sub v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    smin v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    movi v1.2d, #0xffffffffffffffff
+; CHECK-NEXT:    smax v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i1> @llvm.ssub.sat.v16i1(<16 x i1> %x, <16 x i1> %y)
   ret <16 x i1> %z
 }
 
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:

llvm/test/CodeGen/AArch64/uadd_sat.ll

 ; CHECK-NEXT:    ret
   %tmp = call i64 @llvm.uadd.sat.i64(i64 %x, i64 %y);
   ret i64 %tmp;
 }
 
 define i16 @func16(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: func16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #16
-; CHECK-NEXT:    adds w8, w8, w1, lsl #16
-; CHECK-NEXT:    csinv w8, w8, wzr, lo
-; CHECK-NEXT:    lsr w0, w8, #16
+; CHECK-NEXT:    add w8, w0, w1
+; CHECK-NEXT:    mov w9, #65535
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    csel w0, w8, w9, lo
 ; CHECK-NEXT:    ret
   %tmp = call i16 @llvm.uadd.sat.i16(i16 %x, i16 %y);
   ret i16 %tmp;
 }
 
 define i8 @func8(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: func8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #24
-; CHECK-NEXT:    adds w8, w8, w1, lsl #24
-; CHECK-NEXT:    csinv w8, w8, wzr, lo
-; CHECK-NEXT:    lsr w0, w8, #24
+; CHECK-NEXT:    add w8, w0, w1
+; CHECK-NEXT:    cmp w8, #255 // =255
+; CHECK-NEXT:    mov w9, #255
+; CHECK-NEXT:    csel w0, w8, w9, lo
 ; CHECK-NEXT:    ret
   %tmp = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y);
   ret i8 %tmp;
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; CHECK-LABEL: func3:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #28
-; CHECK-NEXT:    adds w8, w8, w1, lsl #28
-; CHECK-NEXT:    csinv w8, w8, wzr, lo
-; CHECK-NEXT:    lsr w0, w8, #28
+; CHECK-NEXT:    add w8, w0, w1
+; CHECK-NEXT:    cmp w8, #15 // =15
+; CHECK-NEXT:    mov w9, #15
+; CHECK-NEXT:    csel w0, w8, w9, lo
 ; CHECK-NEXT:    ret
   %tmp = call i4 @llvm.uadd.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
 }

llvm/test/CodeGen/AArch64/uadd_sat_vec.ll

; CHECK-NEXT:    ret
   %z = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> %x, <8 x i8> %y)
   store <8 x i8> %z, <8 x i8>* %pz
   ret void
 }
 
 define void @v4i8(<4 x i8>* %px, <4 x i8>* %py, <4 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v4i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrb w9, [x1]
 ; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    ldrb w11, [x1, #1]
+; CHECK-NEXT:    ldrb w9, [x1]
 ; CHECK-NEXT:    ldrb w10, [x0, #1]
-; CHECK-NEXT:    fmov s1, w9
-; CHECK-NEXT:    ldrb w9, [x1, #2]
+; CHECK-NEXT:    ldrb w11, [x1, #1]
 ; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    ldrb w8, [x0, #2]
-; CHECK-NEXT:    mov v1.h[1], w11
-; CHECK-NEXT:    ldrb w11, [x1, #3]
+; CHECK-NEXT:    ldrb w9, [x1, #2]
 ; CHECK-NEXT:    mov v0.h[1], w10
+; CHECK-NEXT:    mov v1.h[1], w11
 ; CHECK-NEXT:    ldrb w10, [x0, #3]
-; CHECK-NEXT:    mov v1.h[2], w9
+; CHECK-NEXT:    ldrb w11, [x1, #3]
 ; CHECK-NEXT:    mov v0.h[2], w8
-; CHECK-NEXT:    mov v1.h[3], w11
+; CHECK-NEXT:    mov v1.h[2], w9
 ; CHECK-NEXT:    mov v0.h[3], w10
-; CHECK-NEXT:    shl v1.4h, v1.4h, #8
-; CHECK-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-NEXT:    mvn v2.8b, v1.8b
-; CHECK-NEXT:    umin v0.4h, v0.4h, v2.4h
+; CHECK-NEXT:    mov v1.h[3], w11
 ; CHECK-NEXT:    add v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    ushr v0.4h, v0.4h, #8
+; CHECK-NEXT:    movi d1, #0xff00ff00ff00ff
+; CHECK-NEXT:    umin v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    str s0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <4 x i8>, <4 x i8>* %px
   %y = load <4 x i8>, <4 x i8>* %py
   %z = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> %x, <4 x i8> %y)
   store <4 x i8> %z, <4 x i8>* %pz
   ret void
 }
 
 define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v2i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrb w9, [x1]
 ; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    ldrb w11, [x1, #1]
+; CHECK-NEXT:    ldrb w9, [x1]
 ; CHECK-NEXT:    ldrb w10, [x0, #1]
-; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    ldrb w11, [x1, #1]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    mov v1.s[1], w11
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
-; CHECK-NEXT:    shl v1.2s, v1.2s, #24
-; CHECK-NEXT:    shl v0.2s, v0.2s, #24
-; CHECK-NEXT:    mvn v2.8b, v1.8b
-; CHECK-NEXT:    umin v0.2s, v0.2s, v2.2s
+; CHECK-NEXT:    mov v1.s[1], w11
 ; CHECK-NEXT:    add v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    ushr v0.2s, v0.2s, #24
+; CHECK-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-NEXT:    umin v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
 ; CHECK-NEXT:    strb w8, [x2, #1]
 ; CHECK-NEXT:    strb w9, [x2]
 ; CHECK-NEXT:    ret
   %x = load <2 x i8>, <2 x i8>* %px
   %y = load <2 x i8>, <2 x i8>* %py
   %z = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %x, <2 x i8> %y)
; CHECK-NEXT:    ret
   %z = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> %x, <4 x i16> %y)
   store <4 x i16> %z, <4 x i16>* %pz
   ret void
 }
 
 define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 ; CHECK-LABEL: v2i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrh w9, [x1]
 ; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    ldrh w11, [x1, #2]
+; CHECK-NEXT:    ldrh w9, [x1]
 ; CHECK-NEXT:    ldrh w10, [x0, #2]
-; CHECK-NEXT:    fmov s1, w9
+; CHECK-NEXT:    ldrh w11, [x1, #2]
 ; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    mov v1.s[1], w11
+; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
-; CHECK-NEXT:    shl v1.2s, v1.2s, #16
-; CHECK-NEXT:    shl v0.2s, v0.2s, #16
-; CHECK-NEXT:    mvn v2.8b, v1.8b
-; CHECK-NEXT:    umin v0.2s, v0.2s, v2.2s
+; CHECK-NEXT:    mov v1.s[1], w11
 ; CHECK-NEXT:    add v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    ushr v0.2s, v0.2s, #16
+; CHECK-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-NEXT:    umin v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
 ; CHECK-NEXT:    strh w8, [x2, #2]
 ; CHECK-NEXT:    strh w9, [x2]
 ; CHECK-NEXT:    ret
   %x = load <2 x i16>, <2 x i16>* %px
   %y = load <2 x i16>, <2 x i16>* %py
   %z = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %x, <2 x i16> %y)
; CHECK-NEXT:    ret
   %z = call <1 x i16> @llvm.uadd.sat.v1i16(<1 x i16> %x, <1 x i16> %y)
   store <1 x i16> %z, <1 x i16>* %pz
   ret void
 }
 
 define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; CHECK-LABEL: v16i4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    shl v1.16b, v1.16b, #4
-; CHECK-NEXT:    shl v0.16b, v0.16b, #4
-; CHECK-NEXT:    mvn v2.16b, v1.16b
-; CHECK-NEXT:    umin v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    add v0.16b, v0.16b, v1.16b
-; CHECK-NEXT:    ushr v0.16b, v0.16b, #4
+; CHECK-NEXT:    movi v1.16b, #15
+; CHECK-NEXT:    umin v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i4> @llvm.uadd.sat.v16i4(<16 x i4> %x, <16 x i4> %y)
   ret <16 x i4> %z
 }
 
 define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 ; CHECK-LABEL: v16i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    shl v1.16b, v1.16b, #7
-; CHECK-NEXT:    shl v0.16b, v0.16b, #7
-; CHECK-NEXT:    mvn v2.16b, v1.16b
-; CHECK-NEXT:    umin v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    add v0.16b, v0.16b, v1.16b
-; CHECK-NEXT:    ushr v0.16b, v0.16b, #7
+; CHECK-NEXT:    movi v1.16b, #1
+; CHECK-NEXT:    umin v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i1> @llvm.uadd.sat.v16i1(<16 x i1> %x, <16 x i1> %y)
   ret <16 x i1> %z
 }
 
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:

llvm/test/CodeGen/AArch64/usub_sat.ll

 ; CHECK-NEXT:    ret
   %tmp = call i64 @llvm.usub.sat.i64(i64 %x, i64 %y);
   ret i64 %tmp;
 }
 
 define i16 @func16(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: func16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #16
-; CHECK-NEXT:    subs w8, w8, w1, lsl #16
-; CHECK-NEXT:    csel w8, wzr, w8, lo
-; CHECK-NEXT:    lsr w0, w8, #16
+; CHECK-NEXT:    cmp w0, w1
+; CHECK-NEXT:    csel w8, w0, w1, hi
+; CHECK-NEXT:    sub w0, w8, w1
 ; CHECK-NEXT:    ret
   %tmp = call i16 @llvm.usub.sat.i16(i16 %x, i16 %y);
   ret i16 %tmp;
 }
 
 define i8 @func8(i8 %x, i8 %y) nounwind {
 ; CHECK-LABEL: func8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #24
-; CHECK-NEXT:    subs w8, w8, w1, lsl #24
-; CHECK-NEXT:    csel w8, wzr, w8, lo
-; CHECK-NEXT:    lsr w0, w8, #24
+; CHECK-NEXT:    cmp w0, w1
+; CHECK-NEXT:    csel w8, w0, w1, hi
+; CHECK-NEXT:    sub w0, w8, w1
 ; CHECK-NEXT:    ret
   %tmp = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y);
   ret i8 %tmp;
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; CHECK-LABEL: func3:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsl w8, w0, #28
-; CHECK-NEXT:    subs w8, w8, w1, lsl #28
-; CHECK-NEXT:    csel w8, wzr, w8, lo
-; CHECK-NEXT:    lsr w0, w8, #28
+; CHECK-NEXT:    cmp w0, w1
+; CHECK-NEXT:    csel w8, w0, w1, hi
+; CHECK-NEXT:    sub w0, w8, w1
 ; CHECK-NEXT:    ret
   %tmp = call i4 @llvm.usub.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
 }

llvm/test/CodeGen/AArch64/usub_sat_vec.ll

 ; CHECK-NEXT:    mov v0.h[1], w10
 ; CHECK-NEXT:    mov v1.h[1], w11
 ; CHECK-NEXT:    ldrb w10, [x0, #3]
 ; CHECK-NEXT:    ldrb w11, [x1, #3]
 ; CHECK-NEXT:    mov v0.h[2], w8
 ; CHECK-NEXT:    mov v1.h[2], w9
 ; CHECK-NEXT:    mov v0.h[3], w10
 ; CHECK-NEXT:    mov v1.h[3], w11
-; CHECK-NEXT:    shl v1.4h, v1.4h, #8
-; CHECK-NEXT:    shl v0.4h, v0.4h, #8
 ; CHECK-NEXT:    umax v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    sub v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    ushr v0.4h, v0.4h, #8
 ; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    str s0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <4 x i8>, <4 x i8>* %px
   %y = load <4 x i8>, <4 x i8>* %py
   %z = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> %x, <4 x i8> %y)
   store <4 x i8> %z, <4 x i8>* %pz
   ret void
 }
 
 define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 ; CHECK-LABEL: v2i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldrb w8, [x0]
 ; CHECK-NEXT:    ldrb w9, [x1]
 ; CHECK-NEXT:    ldrb w10, [x0, #1]
 ; CHECK-NEXT:    ldrb w11, [x1, #1]
 ; CHECK-NEXT:    fmov s0, w8
 ; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
 ; CHECK-NEXT:    mov v1.s[1], w11
-; CHECK-NEXT:    shl v1.2s, v1.2s, #24
-; CHECK-NEXT:    shl v0.2s, v0.2s, #24
 ; CHECK-NEXT:    umax v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    sub v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    ushr v0.2s, v0.2s, #24
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
 ; CHECK-NEXT:    strb w8, [x2, #1]
 ; CHECK-NEXT:    strb w9, [x2]
 ; CHECK-NEXT:    ret
   %x = load <2 x i8>, <2 x i8>* %px
   %y = load <2 x i8>, <2 x i8>* %py
   %z = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %x, <2 x i8> %y)

 ; CHECK-NEXT:    ldrh w8, [x0]
 ; CHECK-NEXT:    ldrh w9, [x1]
 ; CHECK-NEXT:    ldrh w10, [x0, #2]
 ; CHECK-NEXT:    ldrh w11, [x1, #2]
 ; CHECK-NEXT:    fmov s0, w8
 ; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    mov v0.s[1], w10
 ; CHECK-NEXT:    mov v1.s[1], w11
-; CHECK-NEXT:    shl v1.2s, v1.2s, #16
-; CHECK-NEXT:    shl v0.2s, v0.2s, #16
 ; CHECK-NEXT:    umax v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    sub v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    ushr v0.2s, v0.2s, #16
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    fmov w9, s0
 ; CHECK-NEXT:    strh w8, [x2, #2]
 ; CHECK-NEXT:    strh w9, [x2]
 ; CHECK-NEXT:    ret
   %x = load <2 x i16>, <2 x i16>* %px
   %y = load <2 x i16>, <2 x i16>* %py
   %z = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %x, <2 x i16> %y)
; CHECK-NEXT:    ret
   %z = call <1 x i16> @llvm.usub.sat.v1i16(<1 x i16> %x, <1 x i16> %y)
   store <1 x i16> %z, <1 x i16>* %pz
   ret void
 }
 
 define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; CHECK-LABEL: v16i4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    shl v1.16b, v1.16b, #4
-; CHECK-NEXT:    shl v0.16b, v0.16b, #4
 ; CHECK-NEXT:    umax v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    sub v0.16b, v0.16b, v1.16b
-; CHECK-NEXT:    ushr v0.16b, v0.16b, #4
 ; CHECK-NEXT:    ret
   %z = call <16 x i4> @llvm.usub.sat.v16i4(<16 x i4> %x, <16 x i4> %y)
   ret <16 x i4> %z
 }
 
 define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 ; CHECK-LABEL: v16i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    shl v1.16b, v1.16b, #7
-; CHECK-NEXT:    shl v0.16b, v0.16b, #7
 ; CHECK-NEXT:    umax v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    sub v0.16b, v0.16b, v1.16b
-; CHECK-NEXT:    ushr v0.16b, v0.16b, #7
 ; CHECK-NEXT:    ret
   %z = call <16 x i1> @llvm.usub.sat.v16i1(<16 x i1> %x, <16 x i1> %y)
   ret <16 x i1> %z
 }
 
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:

llvm/test/CodeGen/ARM/sadd_sat.ll

 ; CHECK-ARM-NEXT:    pop {r11, pc}
   %tmp = call i64 @llvm.sadd.sat.i64(i64 %x, i64 %y)
   ret i64 %tmp
 }
 
 define i16 @func16(i16 %x, i16 %y) nounwind {
 ; CHECK-T1-LABEL: func16:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    lsls r3, r1, #16
-; CHECK-T1-NEXT:    lsls r1, r0, #16
-; CHECK-T1-NEXT:    movs r2, #1
-; CHECK-T1-NEXT:    adds r0, r1, r3
-; CHECK-T1-NEXT:    mov r3, r2
-; CHECK-T1-NEXT:    bmi .LBB2_2
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_0
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    blt .LBB2_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    movs r3, #0
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB2_2:
-; CHECK-T1-NEXT:    cmp r3, #0
-; CHECK-T1-NEXT:    bne .LBB2_4
-; CHECK-T1-NEXT:  @ %bb.3:
-; CHECK-T1-NEXT:    lsls r2, r2, #31
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_1
 ; CHECK-T1-NEXT:    cmp r0, r1
-; CHECK-T1-NEXT:    bvs .LBB2_5
-; CHECK-T1-NEXT:    b .LBB2_6
+; CHECK-T1-NEXT:    bgt .LBB2_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB2_4:
-; CHECK-T1-NEXT:    ldr r2, .LCPI2_0
-; CHECK-T1-NEXT:    cmp r0, r1
-; CHECK-T1-NEXT:    bvc .LBB2_6
-; CHECK-T1-NEXT:  .LBB2_5:
-; CHECK-T1-NEXT:    mov r0, r2
-; CHECK-T1-NEXT:  .LBB2_6:
-; CHECK-T1-NEXT:    asrs r0, r0, #16
 ; CHECK-T1-NEXT:    bx lr
 ; CHECK-T1-NEXT:    .p2align 2
-; CHECK-T1-NEXT:  @ %bb.7:
+; CHECK-T1-NEXT:  @ %bb.5:
 ; CHECK-T1-NEXT:  .LCPI2_0:
-; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
+; CHECK-T1-NEXT:    .long 32767 @ 0x7fff
+; CHECK-T1-NEXT:  .LCPI2_1:
+; CHECK-T1-NEXT:    .long 4294934528 @ 0xffff8000
 ;
 ; CHECK-T2-LABEL: func16:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    lsls r2, r0, #16
-; CHECK-T2-NEXT:    add.w r1, r2, r1, lsl #16
-; CHECK-T2-NEXT:    movs r2, #0
-; CHECK-T2-NEXT:    cmp r1, #0
-; CHECK-T2-NEXT:    mov.w r3, #-2147483648
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi r2, #1
-; CHECK-T2-NEXT:    cmp r2, #0
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    mvnne r3, #-2147483648
-; CHECK-T2-NEXT:    cmp.w r1, r0, lsl #16
-; CHECK-T2-NEXT:    it vc
-; CHECK-T2-NEXT:    movvc r3, r1
-; CHECK-T2-NEXT:    asrs r0, r3, #16
+; CHECK-T2-NEXT:    add r0, r1
+; CHECK-T2-NEXT:    movw r1, #32767
+; CHECK-T2-NEXT:    cmp r0, r1
+; CHECK-T2-NEXT:    it lt
+; CHECK-T2-NEXT:    movlt r1, r0
+; CHECK-T2-NEXT:    movw r0, #32768
+; CHECK-T2-NEXT:    cmn.w r1, #32768
+; CHECK-T2-NEXT:    movt r0, #65535
+; CHECK-T2-NEXT:    it gt
+; CHECK-T2-NEXT:    movgt r0, r1
 ; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func16:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    lsl r2, r0, #16
-; CHECK-ARM-NEXT:    add r1, r2, r1, lsl #16
-; CHECK-ARM-NEXT:    mov r2, #0
-; CHECK-ARM-NEXT:    cmp r1, #0
-; CHECK-ARM-NEXT:    movwmi r2, #1
-; CHECK-ARM-NEXT:    mov r3, #-2147483648
-; CHECK-ARM-NEXT:    cmp r2, #0
-; CHECK-ARM-NEXT:    mvnne r3, #-2147483648
-; CHECK-ARM-NEXT:    cmp r1, r0, lsl #16
-; CHECK-ARM-NEXT:    movvc r3, r1
-; CHECK-ARM-NEXT:    asr r0, r3, #16
+; CHECK-ARM-NEXT:    add r0, r0, r1
+; CHECK-ARM-NEXT:    movw r1, #32767
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    movlt r1, r0
+; CHECK-ARM-NEXT:    movw r0, #32768
+; CHECK-ARM-NEXT:    movt r0, #65535
+; CHECK-ARM-NEXT:    cmn r1, #32768
+; CHECK-ARM-NEXT:    movgt r0, r1
 ; CHECK-ARM-NEXT:    bx lr
   %tmp = call i16 @llvm.sadd.sat.i16(i16 %x, i16 %y)
   ret i16 %tmp
 }
 
 define i8 @func8(i8 %x, i8 %y) nounwind {
 ; CHECK-T1-LABEL: func8:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    lsls r3, r1, #24
-; CHECK-T1-NEXT:    lsls r1, r0, #24
-; CHECK-T1-NEXT:    movs r2, #1
-; CHECK-T1-NEXT:    adds r0, r1, r3
-; CHECK-T1-NEXT:    mov r3, r2
-; CHECK-T1-NEXT:    bmi .LBB3_2
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    movs r1, #127
+; CHECK-T1-NEXT:    cmp r0, #127
+; CHECK-T1-NEXT:    blt .LBB3_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    movs r3, #0
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB3_2:
-; CHECK-T1-NEXT:    cmp r3, #0
-; CHECK-T1-NEXT:    bne .LBB3_4
-; CHECK-T1-NEXT:  @ %bb.3:
-; CHECK-T1-NEXT:    lsls r2, r2, #31
+; CHECK-T1-NEXT:    mvns r1, r1
 ; CHECK-T1-NEXT:    cmp r0, r1
-; CHECK-T1-NEXT:    bvs .LBB3_5
-; CHECK-T1-NEXT:    b .LBB3_6
+; CHECK-T1-NEXT:    bgt .LBB3_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB3_4:
-; CHECK-T1-NEXT:    ldr r2, .LCPI3_0
-; CHECK-T1-NEXT:    cmp r0, r1
-; CHECK-T1-NEXT:    bvc .LBB3_6
-; CHECK-T1-NEXT:  .LBB3_5:
-; CHECK-T1-NEXT:    mov r0, r2
-; CHECK-T1-NEXT:  .LBB3_6:
-; CHECK-T1-NEXT:    asrs r0, r0, #24
 ; CHECK-T1-NEXT:    bx lr
-; CHECK-T1-NEXT:    .p2align 2
-; CHECK-T1-NEXT:  @ %bb.7:
-; CHECK-T1-NEXT:  .LCPI3_0:
-; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
 ; CHECK-T2-LABEL: func8:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    lsls r2, r0, #24
-; CHECK-T2-NEXT:    add.w r1, r2, r1, lsl #24
-; CHECK-T2-NEXT:    movs r2, #0
-; CHECK-T2-NEXT:    cmp r1, #0
-; CHECK-T2-NEXT:    mov.w r3, #-2147483648
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi r2, #1
-; CHECK-T2-NEXT:    cmp r2, #0
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    mvnne r3, #-2147483648
-; CHECK-T2-NEXT:    cmp.w r1, r0, lsl #24
-; CHECK-T2-NEXT:    it vc
-; CHECK-T2-NEXT:    movvc r3, r1
-; CHECK-T2-NEXT:    asrs r0, r3, #24
+; CHECK-T2-NEXT:    add r0, r1
+; CHECK-T2-NEXT:    cmp r0, #127
+; CHECK-T2-NEXT:    it ge
+; CHECK-T2-NEXT:    movge r0, #127
+; CHECK-T2-NEXT:    cmn.w r0, #128
+; CHECK-T2-NEXT:    it le
+; CHECK-T2-NEXT:    mvnle r0, #127
 ; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func8:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    lsl r2, r0, #24
-; CHECK-ARM-NEXT:    add r1, r2, r1, lsl #24
-; CHECK-ARM-NEXT:    mov r2, #0
-; CHECK-ARM-NEXT:    cmp r1, #0
-; CHECK-ARM-NEXT:    movwmi r2, #1
-; CHECK-ARM-NEXT:    mov r3, #-2147483648
-; CHECK-ARM-NEXT:    cmp r2, #0
-; CHECK-ARM-NEXT:    mvnne r3, #-2147483648
-; CHECK-ARM-NEXT:    cmp r1, r0, lsl #24
-; CHECK-ARM-NEXT:    movvc r3, r1
-; CHECK-ARM-NEXT:    asr r0, r3, #24
+; CHECK-ARM-NEXT:    add r0, r0, r1
+; CHECK-ARM-NEXT:    cmp r0, #127
+; CHECK-ARM-NEXT:    movge r0, #127
+; CHECK-ARM-NEXT:    cmn r0, #128
+; CHECK-ARM-NEXT:    mvnle r0, #127
 ; CHECK-ARM-NEXT:    bx lr
   %tmp = call i8 @llvm.sadd.sat.i8(i8 %x, i8 %y)
   ret i8 %tmp
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; CHECK-T1-LABEL: func3:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    lsls r3, r1, #28
-; CHECK-T1-NEXT:    lsls r1, r0, #28
-; CHECK-T1-NEXT:    movs r2, #1
-; CHECK-T1-NEXT:    adds r0, r1, r3
-; CHECK-T1-NEXT:    mov r3, r2
-; CHECK-T1-NEXT:    bmi .LBB4_2
+; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    movs r1, #7
+; CHECK-T1-NEXT:    cmp r0, #7
+; CHECK-T1-NEXT:    blt .LBB4_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    movs r3, #0
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB4_2:
-; CHECK-T1-NEXT:    cmp r3, #0
-; CHECK-T1-NEXT:    bne .LBB4_4
-; CHECK-T1-NEXT:  @ %bb.3:
-; CHECK-T1-NEXT:    lsls r2, r2, #31
+; CHECK-T1-NEXT:    mvns r1, r1
 ; CHECK-T1-NEXT:    cmp r0, r1
-; CHECK-T1-NEXT:    bvs .LBB4_5
-; CHECK-T1-NEXT:    b .LBB4_6
+; CHECK-T1-NEXT:    bgt .LBB4_4
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB4_4:
-; CHECK-T1-NEXT:    ldr r2, .LCPI4_0
-; CHECK-T1-NEXT:    cmp r0, r1
-; CHECK-T1-NEXT:    bvc .LBB4_6
-; CHECK-T1-NEXT:  .LBB4_5:
-; CHECK-T1-NEXT:    mov r0, r2
-; CHECK-T1-NEXT:  .LBB4_6:
-; CHECK-T1-NEXT:    asrs r0, r0, #28
 ; CHECK-T1-NEXT:    bx lr
-; CHECK-T1-NEXT:    .p2align 2
-; CHECK-T1-NEXT:  @ %bb.7:
-; CHECK-T1-NEXT:  .LCPI4_0:
-; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
 ; CHECK-T2-LABEL: func3:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    lsls r2, r0, #28
-; CHECK-T2-NEXT:    add.w r1, r2, r1, lsl #28
-; CHECK-T2-NEXT:    movs r2, #0
-; CHECK-T2-NEXT:    cmp r1, #0
-; CHECK-T2-NEXT:    mov.w r3, #-2147483648
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi r2, #1
-; CHECK-T2-NEXT:    cmp r2, #0
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    mvnne r3, #-2147483648
-; CHECK-T2-NEXT:    cmp.w r1, r0, lsl #28
-; CHECK-T2-NEXT:    it vc
-; CHECK-T2-NEXT:    movvc r3, r1
-; CHECK-T2-NEXT:    asrs r0, r3, #28
+; CHECK-T2-NEXT:    add r0, r1
+; CHECK-T2-NEXT:    cmp r0, #7
+; CHECK-T2-NEXT:    it ge
+; CHECK-T2-NEXT:    movge r0, #7
+; CHECK-T2-NEXT:    cmn.w r0, #8
+; CHECK-T2-NEXT:    it le
+; CHECK-T2-NEXT:    mvnle r0, #7
 ; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func3:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    lsl r2, r0, #28
-; CHECK-ARM-NEXT:    add r1, r2, r1, lsl #28
-; CHECK-ARM-NEXT:    mov r2, #0
-; CHECK-ARM-NEXT:    cmp r1, #0
-; CHECK-ARM-NEXT:    movwmi r2, #1
-; CHECK-ARM-NEXT:    mov r3, #-2147483648
-; CHECK-ARM-NEXT:    cmp r2, #0
-; CHECK-ARM-NEXT:    mvnne r3, #-2147483648
-; CHECK-ARM-NEXT:    cmp r1, r0, lsl #28
-; CHECK-ARM-NEXT:    movvc r3, r1
-; CHECK-ARM-NEXT:    asr r0, r3, #28
+; CHECK-ARM-NEXT:    add r0, r0, r1
+; CHECK-ARM-NEXT:    cmp r0, #7
+; CHECK-ARM-NEXT:    movge r0, #7
+; CHECK-ARM-NEXT:    cmn r0, #8
+; CHECK-ARM-NEXT:    mvnle r0, #7
 ; CHECK-ARM-NEXT:    bx lr
   %tmp = call i4 @llvm.sadd.sat.i4(i4 %x, i4 %y)
   ret i4 %tmp
 }

llvm/test/CodeGen/ARM/ssub_sat.ll

 ; CHECK-ARM-NEXT:    pop {r4, pc}
   %tmp = call i64 @llvm.ssub.sat.i64(i64 %x, i64 %y)
   ret i64 %tmp
 }
 
 define i16 @func16(i16 %x, i16 %y) nounwind {
 ; CHECK-T1-LABEL: func16:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    .save {r4, lr}
-; CHECK-T1-NEXT:    push {r4, lr}
-; CHECK-T1-NEXT:    lsls r1, r1, #16
-; CHECK-T1-NEXT:    lsls r2, r0, #16
-; CHECK-T1-NEXT:    movs r3, #1
-; CHECK-T1-NEXT:    subs r0, r2, r1
-; CHECK-T1-NEXT:    mov r4, r3
-; CHECK-T1-NEXT:    bmi .LBB2_2
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_0
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    blt .LBB2_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    movs r4, #0
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB2_2:
-; CHECK-T1-NEXT:    cmp r4, #0
-; CHECK-T1-NEXT:    bne .LBB2_4
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB2_4
 ; CHECK-T1-NEXT:  @ %bb.3:
-; CHECK-T1-NEXT:    lsls r3, r3, #31
-; CHECK-T1-NEXT:    cmp r2, r1
-; CHECK-T1-NEXT:    bvs .LBB2_5
-; CHECK-T1-NEXT:    b .LBB2_6
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB2_4:
-; CHECK-T1-NEXT:    ldr r3, .LCPI2_0
-; CHECK-T1-NEXT:    cmp r2, r1
-; CHECK-T1-NEXT:    bvc .LBB2_6
-; CHECK-T1-NEXT:  .LBB2_5:
-; CHECK-T1-NEXT:    mov r0, r3
-; CHECK-T1-NEXT:  .LBB2_6:
-; CHECK-T1-NEXT:    asrs r0, r0, #16
-; CHECK-T1-NEXT:    pop {r4, pc}
+; CHECK-T1-NEXT:    bx lr
 ; CHECK-T1-NEXT:    .p2align 2
-; CHECK-T1-NEXT:  @ %bb.7:
+; CHECK-T1-NEXT:  @ %bb.5:
 ; CHECK-T1-NEXT:  .LCPI2_0:
-; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
+; CHECK-T1-NEXT:    .long 32767 @ 0x7fff
+; CHECK-T1-NEXT:  .LCPI2_1:
+; CHECK-T1-NEXT:    .long 4294934528 @ 0xffff8000
 ;
 ; CHECK-T2-LABEL: func16:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    lsls r0, r0, #16
-; CHECK-T2-NEXT:    sub.w r12, r0, r1, lsl #16
-; CHECK-T2-NEXT:    movs r3, #0
-; CHECK-T2-NEXT:    cmp.w r12, #0
-; CHECK-T2-NEXT:    mov.w r2, #-2147483648
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi r3, #1
-; CHECK-T2-NEXT:    cmp r3, #0
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    mvnne r2, #-2147483648
-; CHECK-T2-NEXT:    cmp.w r0, r1, lsl #16
-; CHECK-T2-NEXT:    it vc
-; CHECK-T2-NEXT:    movvc r2, r12
-; CHECK-T2-NEXT:    asrs r0, r2, #16
+; CHECK-T2-NEXT:    subs r0, r0, r1
+; CHECK-T2-NEXT:    movw r1, #32767
+; CHECK-T2-NEXT:    cmp r0, r1
+; CHECK-T2-NEXT:    it lt
+; CHECK-T2-NEXT:    movlt r1, r0
+; CHECK-T2-NEXT:    movw r0, #32768
+; CHECK-T2-NEXT:    cmn.w r1, #32768
+; CHECK-T2-NEXT:    movt r0, #65535
+; CHECK-T2-NEXT:    it gt
+; CHECK-T2-NEXT:    movgt r0, r1
 ; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func16:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    lsl r0, r0, #16
-; CHECK-ARM-NEXT:    sub r12, r0, r1, lsl #16
-; CHECK-ARM-NEXT:    mov r3, #0
-; CHECK-ARM-NEXT:    cmp r12, #0
-; CHECK-ARM-NEXT:    movwmi r3, #1
-; CHECK-ARM-NEXT:    mov r2, #-2147483648
-; CHECK-ARM-NEXT:    cmp r3, #0
-; CHECK-ARM-NEXT:    mvnne r2, #-2147483648
-; CHECK-ARM-NEXT:    cmp r0, r1, lsl #16
-; CHECK-ARM-NEXT:    movvc r2, r12
-; CHECK-ARM-NEXT:    asr r0, r2, #16
+; CHECK-ARM-NEXT:    sub r0, r0, r1
+; CHECK-ARM-NEXT:    movw r1, #32767
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    movlt r1, r0
+; CHECK-ARM-NEXT:    movw r0, #32768
+; CHECK-ARM-NEXT:    movt r0, #65535
+; CHECK-ARM-NEXT:    cmn r1, #32768
+; CHECK-ARM-NEXT:    movgt r0, r1
 ; CHECK-ARM-NEXT:    bx lr
   %tmp = call i16 @llvm.ssub.sat.i16(i16 %x, i16 %y)
   ret i16 %tmp
 }
 
 define i8 @func8(i8 %x, i8 %y) nounwind {
 ; CHECK-T1-LABEL: func8:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    .save {r4, lr}
-; CHECK-T1-NEXT:    push {r4, lr}
-; CHECK-T1-NEXT:    lsls r1, r1, #24
-; CHECK-T1-NEXT:    lsls r2, r0, #24
-; CHECK-T1-NEXT:    movs r3, #1
-; CHECK-T1-NEXT:    subs r0, r2, r1
-; CHECK-T1-NEXT:    mov r4, r3
-; CHECK-T1-NEXT:    bmi .LBB3_2
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    movs r1, #127
+; CHECK-T1-NEXT:    cmp r0, #127
+; CHECK-T1-NEXT:    blt .LBB3_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    movs r4, #0
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB3_2:
-; CHECK-T1-NEXT:    cmp r4, #0
-; CHECK-T1-NEXT:    bne .LBB3_4
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB3_4
 ; CHECK-T1-NEXT:  @ %bb.3:
-; CHECK-T1-NEXT:    lsls r3, r3, #31
-; CHECK-T1-NEXT:    cmp r2, r1
-; CHECK-T1-NEXT:    bvs .LBB3_5
-; CHECK-T1-NEXT:    b .LBB3_6
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB3_4:
-; CHECK-T1-NEXT:    ldr r3, .LCPI3_0
-; CHECK-T1-NEXT:    cmp r2, r1
-; CHECK-T1-NEXT:    bvc .LBB3_6
-; CHECK-T1-NEXT:  .LBB3_5:
-; CHECK-T1-NEXT:    mov r0, r3
-; CHECK-T1-NEXT:  .LBB3_6:
-; CHECK-T1-NEXT:    asrs r0, r0, #24
-; CHECK-T1-NEXT:    pop {r4, pc}
-; CHECK-T1-NEXT:    .p2align 2
-; CHECK-T1-NEXT:  @ %bb.7:
-; CHECK-T1-NEXT:  .LCPI3_0:
-; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
+; CHECK-T1-NEXT:    bx lr
 ;
 ; CHECK-T2-LABEL: func8:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    lsls r0, r0, #24
-; CHECK-T2-NEXT:    sub.w r12, r0, r1, lsl #24
-; CHECK-T2-NEXT:    movs r3, #0
-; CHECK-T2-NEXT:    cmp.w r12, #0
-; CHECK-T2-NEXT:    mov.w r2, #-2147483648
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi r3, #1
-; CHECK-T2-NEXT:    cmp r3, #0
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    mvnne r2, #-2147483648
-; CHECK-T2-NEXT:    cmp.w r0, r1, lsl #24
-; CHECK-T2-NEXT:    it vc
-; CHECK-T2-NEXT:    movvc r2, r12
-; CHECK-T2-NEXT:    asrs r0, r2, #24
+; CHECK-T2-NEXT:    subs r0, r0, r1
+; CHECK-T2-NEXT:    cmp r0, #127
+; CHECK-T2-NEXT:    it ge
+; CHECK-T2-NEXT:    movge r0, #127
+; CHECK-T2-NEXT:    cmn.w r0, #128
+; CHECK-T2-NEXT:    it le
+; CHECK-T2-NEXT:    mvnle r0, #127
 ; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func8:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    lsl r0, r0, #24
-; CHECK-ARM-NEXT:    sub r12, r0, r1, lsl #24
-; CHECK-ARM-NEXT:    mov r3, #0
-; CHECK-ARM-NEXT:    cmp r12, #0
-; CHECK-ARM-NEXT:    movwmi r3, #1
-; CHECK-ARM-NEXT:    mov r2, #-2147483648
-; CHECK-ARM-NEXT:    cmp r3, #0
-; CHECK-ARM-NEXT:    mvnne r2, #-2147483648
-; CHECK-ARM-NEXT:    cmp r0, r1, lsl #24
-; CHECK-ARM-NEXT:    movvc r2, r12
-; CHECK-ARM-NEXT:    asr r0, r2, #24
+; CHECK-ARM-NEXT:    sub r0, r0, r1
+; CHECK-ARM-NEXT:    cmp r0, #127
+; CHECK-ARM-NEXT:    movge r0, #127
+; CHECK-ARM-NEXT:    cmn r0, #128
+; CHECK-ARM-NEXT:    mvnle r0, #127
 ; CHECK-ARM-NEXT:    bx lr
   %tmp = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y)
   ret i8 %tmp
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; CHECK-T1-LABEL: func3:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    .save {r4, lr}
-; CHECK-T1-NEXT:    push {r4, lr}
-; CHECK-T1-NEXT:    lsls r1, r1, #28
-; CHECK-T1-NEXT:    lsls r2, r0, #28
-; CHECK-T1-NEXT:    movs r3, #1
-; CHECK-T1-NEXT:    subs r0, r2, r1
-; CHECK-T1-NEXT:    mov r4, r3
-; CHECK-T1-NEXT:    bmi .LBB4_2
+; CHECK-T1-NEXT:    subs r0, r0, r1
+; CHECK-T1-NEXT:    movs r1, #7
+; CHECK-T1-NEXT:    cmp r0, #7
+; CHECK-T1-NEXT:    blt .LBB4_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    movs r4, #0
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB4_2:
-; CHECK-T1-NEXT:    cmp r4, #0
-; CHECK-T1-NEXT:    bne .LBB4_4
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bgt .LBB4_4
 ; CHECK-T1-NEXT:  @ %bb.3:
-; CHECK-T1-NEXT:    lsls r3, r3, #31
-; CHECK-T1-NEXT:    cmp r2, r1
-; CHECK-T1-NEXT:    bvs .LBB4_5
-; CHECK-T1-NEXT:    b .LBB4_6
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB4_4:
-; CHECK-T1-NEXT:    ldr r3, .LCPI4_0
-; CHECK-T1-NEXT:    cmp r2, r1
-; CHECK-T1-NEXT:    bvc .LBB4_6
-; CHECK-T1-NEXT:  .LBB4_5:
-; CHECK-T1-NEXT:    mov r0, r3
-; CHECK-T1-NEXT:  .LBB4_6:
-; CHECK-T1-NEXT:    asrs r0, r0, #28
-; CHECK-T1-NEXT:    pop {r4, pc}
-; CHECK-T1-NEXT:    .p2align 2
-; CHECK-T1-NEXT:  @ %bb.7:
-; CHECK-T1-NEXT:  .LCPI4_0:
-; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
+; CHECK-T1-NEXT:    bx lr
 ;
 ; CHECK-T2-LABEL: func3:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    lsls r0, r0, #28
-; CHECK-T2-NEXT:    sub.w r12, r0, r1, lsl #28
-; CHECK-T2-NEXT:    movs r3, #0
-; CHECK-T2-NEXT:    cmp.w r12, #0
-; CHECK-T2-NEXT:    mov.w r2, #-2147483648
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi r3, #1
-; CHECK-T2-NEXT:    cmp r3, #0
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    mvnne r2, #-2147483648
-; CHECK-T2-NEXT:    cmp.w r0, r1, lsl #28
-; CHECK-T2-NEXT:    it vc
-; CHECK-T2-NEXT:    movvc r2, r12
-; CHECK-T2-NEXT:    asrs r0, r2, #28
+; CHECK-T2-NEXT:    subs r0, r0, r1
+; CHECK-T2-NEXT:    cmp r0, #7
+; CHECK-T2-NEXT:    it ge
+; CHECK-T2-NEXT:    movge r0, #7
+; CHECK-T2-NEXT:    cmn.w r0, #8
+; CHECK-T2-NEXT:    it le
+; CHECK-T2-NEXT:    mvnle r0, #7
 ; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func3:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    lsl r0, r0, #28
-; CHECK-ARM-NEXT:    sub r12, r0, r1, lsl #28
-; CHECK-ARM-NEXT:    mov r3, #0
-; CHECK-ARM-NEXT:    cmp r12, #0
-; CHECK-ARM-NEXT:    movwmi r3, #1
-; CHECK-ARM-NEXT:    mov r2, #-2147483648
-; CHECK-ARM-NEXT:    cmp r3, #0
-; CHECK-ARM-NEXT:    mvnne r2, #-2147483648
-; CHECK-ARM-NEXT:    cmp r0, r1, lsl #28
-; CHECK-ARM-NEXT:    movvc r2, r12
-; CHECK-ARM-NEXT:    asr r0, r2, #28
+; CHECK-ARM-NEXT:    sub r0, r0, r1
+; CHECK-ARM-NEXT:    cmp r0, #7
+; CHECK-ARM-NEXT:    movge r0, #7
+; CHECK-ARM-NEXT:    cmn r0, #8
+; CHECK-ARM-NEXT:    mvnle r0, #7
 ; CHECK-ARM-NEXT:    bx lr
   %tmp = call i4 @llvm.ssub.sat.i4(i4 %x, i4 %y)
   ret i4 %tmp
 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-T1-LABEL: vec:
 ; CHECK-T1:       @ %bb.0:

llvm/test/CodeGen/ARM/uadd_sat.ll

 ; CHECK-ARM-NEXT:    bx lr
   %tmp = call i64 @llvm.uadd.sat.i64(i64 %x, i64 %y)
   ret i64 %tmp
 }
 
 define i16 @func16(i16 %x, i16 %y) nounwind {
 ; CHECK-T1-LABEL: func16:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    lsls r1, r1, #16
-; CHECK-T1-NEXT:    lsls r0, r0, #16
 ; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    ldr r1, .LCPI2_0
+; CHECK-T1-NEXT:    cmp r0, r1
 ; CHECK-T1-NEXT:    blo .LBB2_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    movs r0, #0
-; CHECK-T1-NEXT:    mvns r0, r0
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB2_2:
-; CHECK-T1-NEXT:    lsrs r0, r0, #16
 ; CHECK-T1-NEXT:    bx lr
+; CHECK-T1-NEXT:    .p2align 2
+; CHECK-T1-NEXT:  @ %bb.3:
+; CHECK-T1-NEXT:  .LCPI2_0:
+; CHECK-T1-NEXT:    .long 65535 @ 0xffff
 ;
 ; CHECK-T2-LABEL: func16:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    lsls r2, r0, #16
-; CHECK-T2-NEXT:    add.w r1, r2, r1, lsl #16
-; CHECK-T2-NEXT:    cmp.w r1, r0, lsl #16
+; CHECK-T2-NEXT:    add r1, r0
+; CHECK-T2-NEXT:    movw r0, #65535
+; CHECK-T2-NEXT:    cmp r1, r0
 ; CHECK-T2-NEXT:    it lo
-; CHECK-T2-NEXT:    movlo.w r1, #-1
-; CHECK-T2-NEXT:    lsrs r0, r1, #16
+; CHECK-T2-NEXT:    movlo r0, r1
 ; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func16:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    lsl r2, r0, #16
-; CHECK-ARM-NEXT:    add r1, r2, r1, lsl #16
-; CHECK-ARM-NEXT:    cmp r1, r0, lsl #16
-; CHECK-ARM-NEXT:    mvnlo r1, #0
-; CHECK-ARM-NEXT:    lsr r0, r1, #16
+; CHECK-ARM-NEXT:    add r1, r0, r1
+; CHECK-ARM-NEXT:    movw r0, #65535
+; CHECK-ARM-NEXT:    cmp r1, r0
+; CHECK-ARM-NEXT:    movlo r0, r1
 ; CHECK-ARM-NEXT:    bx lr
   %tmp = call i16 @llvm.uadd.sat.i16(i16 %x, i16 %y)
   ret i16 %tmp
 }
 
 define i8 @func8(i8 %x, i8 %y) nounwind {
 ; CHECK-T1-LABEL: func8:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    lsls r1, r1, #24
-; CHECK-T1-NEXT:    lsls r0, r0, #24
 ; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    cmp r0, #255
 ; CHECK-T1-NEXT:    blo .LBB3_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    movs r0, #0
-; CHECK-T1-NEXT:    mvns r0, r0
+; CHECK-T1-NEXT:    movs r0, #255
 ; CHECK-T1-NEXT:  .LBB3_2:
-; CHECK-T1-NEXT:    lsrs r0, r0, #24
 ; CHECK-T1-NEXT:    bx lr
 ;
 ; CHECK-T2-LABEL: func8:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    lsls r2, r0, #24
-; CHECK-T2-NEXT:    add.w r1, r2, r1, lsl #24
-; CHECK-T2-NEXT:    cmp.w r1, r0, lsl #24
-; CHECK-T2-NEXT:    it lo
-; CHECK-T2-NEXT:    movlo.w r1, #-1
-; CHECK-T2-NEXT:    lsrs r0, r1, #24
+; CHECK-T2-NEXT:    add r0, r1
+; CHECK-T2-NEXT:    cmp r0, #255
+; CHECK-T2-NEXT:    it hs
+; CHECK-T2-NEXT:    movhs r0, #255
 ; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func8:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    lsl r2, r0, #24
-; CHECK-ARM-NEXT:    add r1, r2, r1, lsl #24
-; CHECK-ARM-NEXT:    cmp r1, r0, lsl #24
-; CHECK-ARM-NEXT:    mvnlo r1, #0
-; CHECK-ARM-NEXT:    lsr r0, r1, #24
+; CHECK-ARM-NEXT:    add r0, r0, r1
+; CHECK-ARM-NEXT:    cmp r0, #255
+; CHECK-ARM-NEXT:    movhs r0, #255
 ; CHECK-ARM-NEXT:    bx lr
   %tmp = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
   ret i8 %tmp
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; CHECK-T1-LABEL: func3:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    lsls r1, r1, #28
-; CHECK-T1-NEXT:    lsls r0, r0, #28
 ; CHECK-T1-NEXT:    adds r0, r0, r1
+; CHECK-T1-NEXT:    cmp r0, #15
 ; CHECK-T1-NEXT:    blo .LBB4_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    movs r0, #0
-; CHECK-T1-NEXT:    mvns r0, r0
+; CHECK-T1-NEXT:    movs r0, #15
 ; CHECK-T1-NEXT:  .LBB4_2:
-; CHECK-T1-NEXT:    lsrs r0, r0, #28
 ; CHECK-T1-NEXT:    bx lr
 ;
 ; CHECK-T2-LABEL: func3:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    lsls r2, r0, #28
-; CHECK-T2-NEXT:    add.w r1, r2, r1, lsl #28
-; CHECK-T2-NEXT:    cmp.w r1, r0, lsl #28
-; CHECK-T2-NEXT:    it lo
-; CHECK-T2-NEXT:    movlo.w r1, #-1
-; CHECK-T2-NEXT:    lsrs r0, r1, #28
+; CHECK-T2-NEXT:    add r0, r1
+; CHECK-T2-NEXT:    cmp r0, #15
+; CHECK-T2-NEXT:    it hs
+; CHECK-T2-NEXT:    movhs r0, #15
 ; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func3:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    lsl r2, r0, #28
-; CHECK-ARM-NEXT:    add r1, r2, r1, lsl #28
-; CHECK-ARM-NEXT:    cmp r1, r0, lsl #28
-; CHECK-ARM-NEXT:    mvnlo r1, #0
-; CHECK-ARM-NEXT:    lsr r0, r1, #28
+; CHECK-ARM-NEXT:    add r0, r0, r1
+; CHECK-ARM-NEXT:    cmp r0, #15
+; CHECK-ARM-NEXT:    movhs r0, #15
 ; CHECK-ARM-NEXT:    bx lr
   %tmp = call i4 @llvm.uadd.sat.i4(i4 %x, i4 %y)
   ret i4 %tmp
 }

llvm/test/CodeGen/ARM/usub_sat.ll

 ; CHECK-ARM-NEXT:    bx lr
   %tmp = call i64 @llvm.usub.sat.i64(i64 %x, i64 %y)
   ret i64 %tmp
 }
 
 define i16 @func16(i16 %x, i16 %y) nounwind {
 ; CHECK-T1-LABEL: func16:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    lsls r1, r1, #16
-; CHECK-T1-NEXT:    lsls r0, r0, #16
-; CHECK-T1-NEXT:    subs r0, r0, r1
-; CHECK-T1-NEXT:    bhs .LBB2_2
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bhi .LBB2_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    movs r0, #0
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB2_2:
-; CHECK-T1-NEXT:    lsrs r0, r0, #16
+; CHECK-T1-NEXT:    subs r0, r0, r1
 ; CHECK-T1-NEXT:    bx lr
 ;
 ; CHECK-T2-LABEL: func16:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    lsls r0, r0, #16
-; CHECK-T2-NEXT:    sub.w r2, r0, r1, lsl #16
-; CHECK-T2-NEXT:    cmp.w r0, r1, lsl #16
-; CHECK-T2-NEXT:    it lo
-; CHECK-T2-NEXT:    movlo r2, #0
-; CHECK-T2-NEXT:    lsrs r0, r2, #16
+; CHECK-T2-NEXT:    cmp r0, r1
+; CHECK-T2-NEXT:    it ls
+; CHECK-T2-NEXT:    movls r0, r1
+; CHECK-T2-NEXT:    subs r0, r0, r1
 ; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func16:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    lsl r0, r0, #16
-; CHECK-ARM-NEXT:    sub r2, r0, r1, lsl #16
-; CHECK-ARM-NEXT:    cmp r0, r1, lsl #16
-; CHECK-ARM-NEXT:    movlo r2, #0
-; CHECK-ARM-NEXT:    lsr r0, r2, #16
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    movls r0, r1
+; CHECK-ARM-NEXT:    sub r0, r0, r1
 ; CHECK-ARM-NEXT:    bx lr
   %tmp = call i16 @llvm.usub.sat.i16(i16 %x, i16 %y)
   ret i16 %tmp
 }
 
 define i8 @func8(i8 %x, i8 %y) nounwind {
 ; CHECK-T1-LABEL: func8:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    lsls r1, r1, #24
-; CHECK-T1-NEXT:    lsls r0, r0, #24
-; CHECK-T1-NEXT:    subs r0, r0, r1
-; CHECK-T1-NEXT:    bhs .LBB3_2
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bhi .LBB3_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    movs r0, #0
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB3_2:
-; CHECK-T1-NEXT:    lsrs r0, r0, #24
+; CHECK-T1-NEXT:    subs r0, r0, r1
 ; CHECK-T1-NEXT:    bx lr
 ;
 ; CHECK-T2-LABEL: func8:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    lsls r0, r0, #24
-; CHECK-T2-NEXT:    sub.w r2, r0, r1, lsl #24
-; CHECK-T2-NEXT:    cmp.w r0, r1, lsl #24
-; CHECK-T2-NEXT:    it lo
-; CHECK-T2-NEXT:    movlo r2, #0
-; CHECK-T2-NEXT:    lsrs r0, r2, #24
+; CHECK-T2-NEXT:    cmp r0, r1
+; CHECK-T2-NEXT:    it ls
+; CHECK-T2-NEXT:    movls r0, r1
+; CHECK-T2-NEXT:    subs r0, r0, r1
 ; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func8:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    lsl r0, r0, #24
-; CHECK-ARM-NEXT:    sub r2, r0, r1, lsl #24
-; CHECK-ARM-NEXT:    cmp r0, r1, lsl #24
-; CHECK-ARM-NEXT:    movlo r2, #0
-; CHECK-ARM-NEXT:    lsr r0, r2, #24
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    movls r0, r1
+; CHECK-ARM-NEXT:    sub r0, r0, r1
 ; CHECK-ARM-NEXT:    bx lr
   %tmp = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y)
   ret i8 %tmp
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; CHECK-T1-LABEL: func3:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    lsls r1, r1, #28
-; CHECK-T1-NEXT:    lsls r0, r0, #28
-; CHECK-T1-NEXT:    subs r0, r0, r1
-; CHECK-T1-NEXT:    bhs .LBB4_2
+; CHECK-T1-NEXT:    cmp r0, r1
+; CHECK-T1-NEXT:    bhi .LBB4_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    movs r0, #0
+; CHECK-T1-NEXT:    mov r0, r1
 ; CHECK-T1-NEXT:  .LBB4_2:
-; CHECK-T1-NEXT:    lsrs r0, r0, #28
+; CHECK-T1-NEXT:    subs r0, r0, r1
 ; CHECK-T1-NEXT:    bx lr
 ;
 ; CHECK-T2-LABEL: func3:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    lsls r0, r0, #28
-; CHECK-T2-NEXT:    sub.w r2, r0, r1, lsl #28
-; CHECK-T2-NEXT:    cmp.w r0, r1, lsl #28
-; CHECK-T2-NEXT:    it lo
-; CHECK-T2-NEXT:    movlo r2, #0
-; CHECK-T2-NEXT:    lsrs r0, r2, #28
+; CHECK-T2-NEXT:    cmp r0, r1
+; CHECK-T2-NEXT:    it ls
+; CHECK-T2-NEXT:    movls r0, r1
+; CHECK-T2-NEXT:    subs r0, r0, r1
 ; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func3:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    lsl r0, r0, #28
-; CHECK-ARM-NEXT:    sub r2, r0, r1, lsl #28
-; CHECK-ARM-NEXT:    cmp r0, r1, lsl #28
-; CHECK-ARM-NEXT:    movlo r2, #0
-; CHECK-ARM-NEXT:    lsr r0, r2, #28
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    movls r0, r1
+; CHECK-ARM-NEXT:    sub r0, r0, r1
 ; CHECK-ARM-NEXT:    bx lr
   %tmp = call i4 @llvm.usub.sat.i4(i4 %x, i4 %y)
   ret i4 %tmp
 }

llvm/test/CodeGen/X86/sadd_sat.ll

; X64-NEXT:    retq
   %tmp = call i8 @llvm.sadd.sat.i8(i8 %x, i8 %y)
   ret i8 %tmp
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; X86-LABEL: func3:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; X86-NEXT:    shlb $4, %dl
-; X86-NEXT:    shlb $4, %al
-; X86-NEXT:    xorl %ecx, %ecx
-; X86-NEXT:    movb %al, %ah
-; X86-NEXT:    addb %dl, %ah
-; X86-NEXT:    setns %cl
-; X86-NEXT:    addl $127, %ecx
-; X86-NEXT:    addb %dl, %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    cmovol %ecx, %eax
-; X86-NEXT:    sarb $4, %al
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movzbl %al, %ecx
+; X86-NEXT:    cmpb $7, %al
+; X86-NEXT:    movl $7, %edx
+; X86-NEXT:    cmovll %ecx, %edx
+; X86-NEXT:    cmpb $-8, %dl
+; X86-NEXT:    movl $248, %eax
+; X86-NEXT:    cmovgl %edx, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: func3:
 ; X64:       # %bb.0:
-; X64-NEXT:    shlb $4, %sil
-; X64-NEXT:    shlb $4, %dil
-; X64-NEXT:    xorl %ecx, %ecx
-; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    addb %sil, %al
-; X64-NEXT:    setns %cl
-; X64-NEXT:    addl $127, %ecx
 ; X64-NEXT:    addb %sil, %dil
 ; X64-NEXT:    movzbl %dil, %eax
-; X64-NEXT:    cmovol %ecx, %eax
-; X64-NEXT:    sarb $4, %al
+; X64-NEXT:    cmpb $7, %al
+; X64-NEXT:    movl $7, %ecx
+; X64-NEXT:    cmovll %eax, %ecx
+; X64-NEXT:    cmpb $-8, %cl
+; X64-NEXT:    movl $248, %eax
+; X64-NEXT:    cmovgl %ecx, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %tmp = call i4 @llvm.sadd.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-LABEL: vec:

llvm/test/CodeGen/X86/ssub_sat.ll

; X64-NEXT:    retq
   %tmp = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y)
   ret i8 %tmp
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; X86-LABEL: func3:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; X86-NEXT:    shlb $4, %dl
-; X86-NEXT:    shlb $4, %al
-; X86-NEXT:    xorl %ecx, %ecx
-; X86-NEXT:    movb %al, %ah
-; X86-NEXT:    subb %dl, %ah
-; X86-NEXT:    setns %cl
-; X86-NEXT:    addl $127, %ecx
-; X86-NEXT:    subb %dl, %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    cmovol %ecx, %eax
-; X86-NEXT:    sarb $4, %al
+; X86-NEXT:    subb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movzbl %al, %ecx
+; X86-NEXT:    cmpb $7, %al
+; X86-NEXT:    movl $7, %edx
+; X86-NEXT:    cmovll %ecx, %edx
+; X86-NEXT:    cmpb $-8, %dl
+; X86-NEXT:    movl $248, %eax
+; X86-NEXT:    cmovgl %edx, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: func3:
 ; X64:       # %bb.0:
-; X64-NEXT:    shlb $4, %sil
-; X64-NEXT:    shlb $4, %dil
-; X64-NEXT:    xorl %ecx, %ecx
-; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    subb %sil, %al
-; X64-NEXT:    setns %cl
-; X64-NEXT:    addl $127, %ecx
 ; X64-NEXT:    subb %sil, %dil
 ; X64-NEXT:    movzbl %dil, %eax
-; X64-NEXT:    cmovol %ecx, %eax
-; X64-NEXT:    sarb $4, %al
+; X64-NEXT:    cmpb $7, %al
+; X64-NEXT:    movl $7, %ecx
+; X64-NEXT:    cmovll %eax, %ecx
+; X64-NEXT:    cmpb $-8, %cl
+; X64-NEXT:    movl $248, %eax
+; X64-NEXT:    cmovgl %ecx, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %tmp = call i4 @llvm.ssub.sat.i4(i4 %x, i4 %y)
   ret i4 %tmp
 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-LABEL: vec:

llvm/test/CodeGen/X86/uadd_sat.ll

; X64-NEXT:    retq
   %tmp = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
   ret i8 %tmp
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; X86-LABEL: func3:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT:    shlb $4, %cl
-; X86-NEXT:    shlb $4, %al
-; X86-NEXT:    addb %cl, %al
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    movzbl %al, %ecx
-; X86-NEXT:    movl $255, %eax
-; X86-NEXT:    cmovael %ecx, %eax
-; X86-NEXT:    shrb $4, %al
+; X86-NEXT:    cmpb $15, %al
+; X86-NEXT:    movl $15, %eax
+; X86-NEXT:    cmovbl %ecx, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: func3:
 ; X64:       # %bb.0:
-; X64-NEXT:    shlb $4, %sil
-; X64-NEXT:    shlb $4, %dil
 ; X64-NEXT:    addb %sil, %dil
 ; X64-NEXT:    movzbl %dil, %ecx
-; X64-NEXT:    movl $255, %eax
-; X64-NEXT:    cmovael %ecx, %eax
-; X64-NEXT:    shrb $4, %al
+; X64-NEXT:    cmpb $15, %cl
+; X64-NEXT:    movl $15, %eax
+; X64-NEXT:    cmovbl %ecx, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %tmp = call i4 @llvm.uadd.sat.i4(i4 %x, i4 %y)
   ret i4 %tmp
 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-LABEL: vec:

llvm/test/CodeGen/X86/usub_sat.ll

 ; X64-NEXT:    retq
   %tmp = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y)
   ret i8 %tmp
 }
 
 define i4 @func3(i4 %x, i4 %y) nounwind {
 ; X86-LABEL: func3:
 ; X86:       # %bb.0:
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT:    shlb $4, %cl
-; X86-NEXT:    shlb $4, %al
-; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    cmpb %cl, %dl
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    cmoval %edx, %eax
 ; X86-NEXT:    subb %cl, %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    cmovbl %edx, %eax
-; X86-NEXT:    shrb $4, %al
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: func3:
 ; X64:       # %bb.0:
-; X64-NEXT:    shlb $4, %sil
-; X64-NEXT:    shlb $4, %dil
-; X64-NEXT:    xorl %ecx, %ecx
-; X64-NEXT:    subb %sil, %dil
-; X64-NEXT:    movzbl %dil, %eax
-; X64-NEXT:    cmovbl %ecx, %eax
-; X64-NEXT:    shrb $4, %al
+; X64-NEXT:    cmpb %sil, %dil
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmoval %edi, %eax
+; X64-NEXT:    subb %sil, %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %tmp = call i4 @llvm.usub.sat.i4(i4 %x, i4 %y)
   ret i4 %tmp
 }
 
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-LABEL: vec: