[ARM] Selects SSAT/USAT from LLVM IR of min/max patterns

[MeeraN]

Summary

LLVM will canonicalize conditional selectors to a different pattern than the old code that was used.
This is updating the function to match the new expected patterns and select SSAT or USAT when successful.
Tests have also been updated to use the new patterns.

[dmgreen]

It's probably worth adding something to the commit message about how llvm will canonicalize to different patterns than the old code would select from, and that this is updating it to the new expected form.

llvm/lib/Target/ARM/ARMISelLowering.cpp

5002

It looks like this isn't used any more. If not, we can remove it. Unfortunately it appears that isLowerSaturate is still used.

5003

Can you add something to this comment about llvm canonicalizing to min(max(..)) or max(min(..))

5021–5022

Can you also change usat to Usat or USat or something like it.

5094–5095

Can you move the comment to its own line, to make the formatting nicer.

You can also possibly reverse the if conditions above to return early and reduce the need for indenting.

[MeeraN]

Added explanation about LLVM canonicalizing to min/max patterns to both the comment before the function and commit message. Tidied up formatting by moving a comment, reducing indention where possible and removing unused function (isUpperSaturate).

[dmgreen]

Nice one. LGTM

[mstorsjo]

Headsup: This broke a number of tests for me. Looking closer into where it changed things erroneously...

[mstorsjo]

The main difference in code, in the function that show the error in one of the testcases that broke, looks like this:

-       movs    r1, #9
-       cmp     r0, #9
-       it      lt
-       movlt   r1, r0
-       bic.w   r0, r1, r1, asr #31
+       usat    r0, #1, r0

[mstorsjo]

The file that shows the error can be built from https://martin.st/temp/eval.c, built as clang -target armv7-w64-mingw32 -c -O2 eval.c.

[MeeraN]

In D87379#2273244, @mstorsjo wrote:

Headsup: This broke a number of tests for me. Looking closer into where it changed things erroneously...

It seems we made a small mistake in one of the if conditions, will fix it now. Thanks for letting us know

[mstorsjo]

In D87379#2273520, @MeeraN wrote:

In D87379#2273244, @mstorsjo wrote:

Headsup: This broke a number of tests for me. Looking closer into where it changed things erroneously...

It seems we made a small mistake in one of the if conditions, will fix it now. Thanks for letting us know

Thanks for the fix; my testset runs fine now again.

[srj]

It seems we made a small mistake in one of the if conditions, will fix it now. Thanks for letting us know

What commit/review has the fixes? This is affecting Halide as well and we'd like to test the fix.

[mstorsjo]

In D87379#2274531, @srj wrote:

It seems we made a small mistake in one of the if conditions, will fix it now. Thanks for letting us know

What commit/review has the fixes? This is affecting Halide as well and we'd like to test the fix.

The fix was in https://github.com/llvm/llvm-project/commit/1119bf95be94950da602b268dc96dbb2110cbe15 / rG1119bf95be94950da602b268dc96dbb2110cbe15

[MeeraN]

Closed by revision https://reviews.llvm.org/rGdd519bf0b074

llvm/lib/Target/ARM/ARMISelLowering.cpp

static bool isLowerSaturate(const SDValue LHS, const SDValue RHS,
                             const SDValue TrueVal, const SDValue FalseVal,
                             const ISD::CondCode CC, const SDValue K) {
   return (isGTorGE(CC) &&
           ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))) ||
          (isLTorLE(CC) &&
           ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal)));
 }
 
-// Similar to isLowerSaturate(), but checks for upper-saturating conditions.
-static bool isUpperSaturate(const SDValue LHS, const SDValue RHS,
-                            const SDValue TrueVal, const SDValue FalseVal,
-                            const ISD::CondCode CC, const SDValue K) {
-  return (isGTorGE(CC) &&
-          ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal))) ||
-         (isLTorLE(CC) &&
-          ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal)));
-}
-
 // Check if two chained conditionals could be converted into SSAT or USAT.
 //
 // SSAT can replace a set of two conditional selectors that bound a number to an
 // interval of type [k, ~k] when k + 1 is a power of 2. Here are some examples:
 //
 //     x < -k ? -k : (x > k ? k : x)
 //     x < -k ? -k : (x < k ? x : k)
 //     x > -k ? (x > k ? k : x) : -k
 //     x < k ? (x < -k ? -k : x) : k
 //     etc.
 //
+// LLVM canonicalizes these to either a min(max()) or a max(min()) pattern.
+// This function tries to match one of these and will return true if successful.
+//
 // USAT works similarily to SSAT but bounds on the interval [0, k] where k + 1 is
 // a power of 2.
 //
 // It returns true if the conversion can be done, false otherwise.
 // Additionally, the variable is returned in parameter V, the constant in K and
 // usat is set to true if the conditional represents an unsigned saturation
 static bool isSaturatingConditional(const SDValue &Op, SDValue &V,
-                                    uint64_t &K, bool &usat) {
-  SDValue LHS1 = Op.getOperand(0);
-  SDValue RHS1 = Op.getOperand(1);
+                                    uint64_t &K, bool &Usat) {
+  SDValue V1 = Op.getOperand(0);
+  SDValue K1 = Op.getOperand(1);
   SDValue TrueVal1 = Op.getOperand(2);
   SDValue FalseVal1 = Op.getOperand(3);
   ISD::CondCode CC1 = cast<CondCodeSDNode>(Op.getOperand(4))->get();
 
   const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
   if (Op2.getOpcode() != ISD::SELECT_CC)
     return false;
 
-  SDValue LHS2 = Op2.getOperand(0);
-  SDValue RHS2 = Op2.getOperand(1);
+  SDValue V2 = Op2.getOperand(0);
+  SDValue K2 = Op2.getOperand(1);
   SDValue TrueVal2 = Op2.getOperand(2);
   SDValue FalseVal2 = Op2.getOperand(3);
   ISD::CondCode CC2 = cast<CondCodeSDNode>(Op2.getOperand(4))->get();
 
-  // Find out which are the constants and which are the variables
-  // in each conditional
-  SDValue *K1 = isa<ConstantSDNode>(LHS1) ? &LHS1 : isa<ConstantSDNode>(RHS1)
-                                                        ? &RHS1
-                                                        : nullptr;
-  SDValue *K2 = isa<ConstantSDNode>(LHS2) ? &LHS2 : isa<ConstantSDNode>(RHS2)
-                                                        ? &RHS2
-                                                        : nullptr;
-  SDValue K2Tmp = isa<ConstantSDNode>(TrueVal2) ? TrueVal2 : FalseVal2;
-  SDValue V1Tmp = (K1 && *K1 == LHS1) ? RHS1 : LHS1;
-  SDValue V2Tmp = (K2 && *K2 == LHS2) ? RHS2 : LHS2;
-  SDValue V2 = (K2Tmp == TrueVal2) ? FalseVal2 : TrueVal2;
-
-  // We must detect cases where the original operations worked with 16- or
-  // 8-bit values. In such case, V2Tmp != V2 because the comparison operations
-  // must work with sign-extended values but the select operations return
-  // the original non-extended value.
-  SDValue V2TmpReg = V2Tmp;
-  if (V2Tmp->getOpcode() == ISD::SIGN_EXTEND_INREG)
-    V2TmpReg = V2Tmp->getOperand(0);
-
-  // Check that the registers and the constants have the correct values
-  // in both conditionals
-  if (!K1 || !K2 || *K1 == Op2 || *K2 != K2Tmp || V1Tmp != V2Tmp ||
-      V2TmpReg != V2)
-    return false;
-
-  // Figure out which conditional is saturating the lower/upper bound.
-  const SDValue *LowerCheckOp =
-      isLowerSaturate(LHS1, RHS1, TrueVal1, FalseVal1, CC1, *K1)
-          ? &Op
-          : isLowerSaturate(LHS2, RHS2, TrueVal2, FalseVal2, CC2, *K2)
-                ? &Op2
-                : nullptr;
-  const SDValue *UpperCheckOp =
-      isUpperSaturate(LHS1, RHS1, TrueVal1, FalseVal1, CC1, *K1)
-          ? &Op
-          : isUpperSaturate(LHS2, RHS2, TrueVal2, FalseVal2, CC2, *K2)
-                ? &Op2
-                : nullptr;
+  SDValue V1Tmp = V1;
+  SDValue V2Tmp = V2;
 
-  if (!UpperCheckOp || !LowerCheckOp || LowerCheckOp == UpperCheckOp)
-    return false;
+  if (V1.getOpcode() == ISD::SIGN_EXTEND_INREG &&
+      V2.getOpcode() == ISD::SIGN_EXTEND_INREG) {
+    V1Tmp = V1.getOperand(0);
+    V2Tmp = V2.getOperand(0);
+  }
+
+  // Check that the registers and the constants match a max(min()) or min(max())
+  // pattern
+  if (V1Tmp == TrueVal1 && V2Tmp == TrueVal2 && K1 == FalseVal1 &&
+      K2 == FalseVal2 &&
+      ((isGTorGE(CC1) && isLTorLE(CC2)) || (isLTorLE(CC1) && isGTorGE(CC2)))) {
 
     // Check that the constant in the lower-bound check is
     // the opposite of the constant in the upper-bound check
     // in 1's complement.
-  int64_t Val1 = cast<ConstantSDNode>(*K1)->getSExtValue();
-  int64_t Val2 = cast<ConstantSDNode>(*K2)->getSExtValue();
+    if (!isa<ConstantSDNode>(K1) || !isa<ConstantSDNode>(K2))
+      return false;
+
+    int64_t Val1 = cast<ConstantSDNode>(K1)->getSExtValue();
+    int64_t Val2 = cast<ConstantSDNode>(K2)->getSExtValue();
     int64_t PosVal = std::max(Val1, Val2);
     int64_t NegVal = std::min(Val1, Val2);
 
-  if (((Val1 > Val2 && UpperCheckOp == &Op) ||
-       (Val1 < Val2 && UpperCheckOp == &Op2)) &&
-      isPowerOf2_64(PosVal + 1)) {
+    if (!((Val1 > Val2 && isLTorLE(CC1)) || (Val1 < Val2 && isLTorLE(CC2))) &&
+        !isPowerOf2_64(PosVal + 1)) 
+      return false;
 
-    // Handle the difference between USAT (unsigned) and SSAT (signed) saturation
+    // Handle the difference between USAT (unsigned) and SSAT (signed)
+    // saturation
     if (Val1 == ~Val2)
-      usat = false;
+      Usat = false;
     else if (NegVal == 0)
-      usat = true;
+      Usat = true;
     else
       return false;
 
-    V = V2;
-    K = (uint64_t)PosVal; // At this point, PosVal is guaranteed to be positive
+    V = V2Tmp;
+    // At this point, PosVal is guaranteed to be positive
+    K = (uint64_t) PosVal; 
 
     return true;
   }
-
   return false;
 }
 
 // Check if a condition of the type x < k ? k : x can be converted into a
 // bit operation instead of conditional moves.
 // Currently this is allowed given:
 // - The conditions and values match up
 // - k is 0 or -1 (all ones)
 // This function will not check the last condition, thats up to the caller
 // It returns true if the transformation can be made, and in such case
 // returns x in V, and k in SatK.
 static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V,
                                          SDValue &SatK)
 {
   SDValue LHS = Op.getOperand(0);
   SDValue RHS = Op.getOperand(1);
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
   SDValue TrueVal = Op.getOperand(2);
   SDValue FalseVal = Op.getOperand(3);
 

llvm/test/CodeGen/ARM/ssat.ll

 
 ; x < -k ? -k : (x > k ? k : x)
 ; 32-bit base test
 define i32 @sat_base_32bit(i32 %x) #0 {
 ; CHECK-LABEL: sat_base_32bit:
 ; V6T2: ssat r0, #24, r0
 ; V4T-NOT: ssat
 entry:
-  %cmpLow = icmp slt i32 %x, -8388608
-  %cmpUp = icmp sgt i32 %x, 8388607
-  %saturateUp = select i1 %cmpUp, i32 8388607, i32 %x
-  %saturateLow = select i1 %cmpLow, i32 -8388608, i32 %saturateUp
+  %0 = icmp slt i32 %x, 8388607
+  %saturateUp = select i1 %0, i32 %x, i32 8388607
+  %1 = icmp sgt i32 %saturateUp, -8388608
+  %saturateLow = select i1 %1, i32 %saturateUp, i32 -8388608
   ret i32 %saturateLow
 }
 
 ; x < -k ? -k : (x > k ? k : x)
 ; 16-bit base test
 define i16 @sat_base_16bit(i16 %x) #0 {
 ; CHECK-LABEL: sat_base_16bit:
 ; V6T2: ssat r0, #12, r0
 ; V4T-NOT: ssat
 entry:
-  %cmpLow = icmp slt i16 %x, -2048
-  %cmpUp = icmp sgt i16 %x, 2047
-  %saturateUp = select i1 %cmpUp, i16 2047, i16 %x
-  %saturateLow = select i1 %cmpLow, i16 -2048, i16 %saturateUp
+  %0 = icmp slt i16 %x, 2047
+  %saturateUp = select i1 %0, i16 %x, i16 2047
+  %1 = icmp sgt i16 %saturateUp, -2048
+  %saturateLow = select i1 %1, i16 %saturateUp, i16 -2048
   ret i16 %saturateLow
 }
 
 ; x < -k ? -k : (x > k ? k : x)
 ; 8-bit base test
 define i8 @sat_base_8bit(i8 %x) #0 {
 ; CHECK-LABEL: sat_base_8bit:
 ; V6T2: ssat r0, #6, r0
 ; V4T-NOT: ssat
 entry:
-  %cmpLow = icmp slt i8 %x, -32
-  %cmpUp = icmp sgt i8 %x, 31
-  %saturateUp = select i1 %cmpUp, i8 31, i8 %x
-  %saturateLow = select i1 %cmpLow, i8 -32, i8 %saturateUp
+  %0 = icmp slt i8 %x, 31
+  %saturateUp = select i1 %0, i8 %x, i8 31
+  %1 = icmp sgt i8 %saturateUp, -32
+  %saturateLow = select i1 %1, i8 %saturateUp, i8 -32
   ret i8 %saturateLow
 }
 
 ;
 ; Tests where the conditionals that check for upper and lower bounds,
 ; or the < and > operators, are arranged in different ways. Only some
 ; of the possible combinations that lead to SSAT are tested.
 ;
 
 ; x < -k ? -k : (x < k ? x : k)
 define i32 @sat_lower_upper_1(i32 %x) #0 {
 ; CHECK-LABEL: sat_lower_upper_1:
 ; V6T2: ssat r0, #24, r0
 ; V4T-NOT: ssat
 entry:
-  %cmpLow = icmp slt i32 %x, -8388608
   %cmpUp = icmp slt i32 %x, 8388607
   %saturateUp = select i1 %cmpUp, i32 %x, i32 8388607
-  %saturateLow = select i1 %cmpLow, i32 -8388608, i32 %saturateUp
+  %0 = icmp sgt i32 %saturateUp, -8388608
+  %saturateLow = select i1 %0, i32 %saturateUp, i32 -8388608
   ret i32 %saturateLow
 }
 
 ; x > -k ? (x > k ? k : x) : -k
 define i32 @sat_lower_upper_2(i32 %x) #0 {
 ; CHECK-LABEL: sat_lower_upper_2:
 ; V6T2: ssat    r0, #24, r0
 ; V4T-NOT: ssat
 entry:
-  %cmpLow = icmp sgt i32 %x, -8388608
-  %cmpUp = icmp sgt i32 %x, 8388607
-  %saturateUp = select i1 %cmpUp, i32 8388607, i32 %x
-  %saturateLow = select i1 %cmpLow, i32 %saturateUp, i32 -8388608
+  %0 = icmp slt i32 %x, 8388607
+  %saturateUp = select i1 %0, i32 %x, i32 8388607
+  %1 = icmp sgt i32 %saturateUp, -8388608
+  %saturateLow = select i1 %1, i32 %saturateUp, i32 -8388608
   ret i32 %saturateLow
 }
 
 ; x < k ? (x < -k ? -k : x) : k
 define i32 @sat_upper_lower_1(i32 %x) #0 {
 ; CHECK-LABEL: sat_upper_lower_1:
 ; V6T2: ssat    r0, #24, r0
 ; V4T-NOT: ssat
 entry:
-  %cmpUp = icmp slt i32 %x, 8388607
-  %cmpLow = icmp slt i32 %x, -8388608
-  %saturateLow = select i1 %cmpLow, i32 -8388608, i32 %x
-  %saturateUp = select i1 %cmpUp, i32 %saturateLow, i32 8388607
+  %0 = icmp sgt i32 %x, -8388608
+  %saturateLow = select i1 %0, i32 %x, i32 -8388608
+  %1 = icmp slt i32 %saturateLow, 8388607
+  %saturateUp = select i1 %1, i32 %saturateLow, i32 8388607
   ret i32 %saturateUp
 }
 
 ; x > k ? k : (x < -k ? -k : x)
 define i32 @sat_upper_lower_2(i32 %x) #0 {
 ; CHECK-LABEL: sat_upper_lower_2:
 ; V6T2: ssat    r0, #24, r0
 ; V4T-NOT: ssat
 entry:
-  %cmpUp = icmp sgt i32 %x, 8388607
-  %cmpLow = icmp slt i32 %x, -8388608
-  %saturateLow = select i1 %cmpLow, i32 -8388608, i32 %x
-  %saturateUp = select i1 %cmpUp, i32 8388607, i32 %saturateLow
+  %0 = icmp sgt i32 %x, -8388608
+  %saturateLow = select i1 %0, i32 %x, i32 -8388608
+  %1 = icmp slt i32 %saturateLow, 8388607
+  %saturateUp = select i1 %1, i32 %saturateLow, i32 8388607
   ret i32 %saturateUp
 }
 
 ; k < x ? k : (x > -k ? x : -k)
 define i32 @sat_upper_lower_3(i32 %x) #0 {
 ; CHECK-LABEL: sat_upper_lower_3:
 ; V6T2: ssat    r0, #24, r0
 ; V4T-NOT: ssat
 entry:
-  %cmpUp = icmp slt i32 8388607, %x
   %cmpLow = icmp sgt i32 %x, -8388608
   %saturateLow = select i1 %cmpLow, i32 %x, i32 -8388608
-  %saturateUp = select i1 %cmpUp, i32 8388607, i32 %saturateLow
+  %0 = icmp slt i32 %saturateLow, 8388607
+  %saturateUp = select i1 %0, i32 %saturateLow, i32 8388607
   ret i32 %saturateUp
 }
 
 ;
 ; Miscellanea
 ;
 
 ; Check that >= and <= work the same as > and <
 ; k <= x ? k : (x >= -k ? x : -k)
 define i32 @sat_le_ge(i32 %x) #0 {
 ; CHECK-LABEL: sat_le_ge:
 ; V6T2: ssat    r0, #24, r0
 ; V4T-NOT: ssat
 entry:
-  %cmpUp = icmp sle i32 8388607, %x
-  %cmpLow = icmp sge i32 %x, -8388608
-  %saturateLow = select i1 %cmpLow, i32 %x, i32 -8388608
-  %saturateUp = select i1 %cmpUp, i32 8388607, i32 %saturateLow
+  %0 = icmp sgt i32 %x, -8388608
+  %saturateLow = select i1 %0, i32 %x, i32 -8388608
+  %1 = icmp slt i32 %saturateLow, 8388607
+  %saturateUp = select i1 %1, i32 %saturateLow, i32 8388607
   ret i32 %saturateUp
 }
 
 ;
 ; The following tests check for patterns that should not transform
 ; into SSAT but are similar enough that could confuse the selector.
 ;
 
 ; x > k ? k : (x > -k ? -k : x)
 ; First condition upper-saturates, second doesn't lower-saturate.
 define i32 @no_sat_missing_lower(i32 %x) #0 {
 ; CHECK-LABEL: no_sat_missing_lower
 ; CHECK-NOT: ssat
 entry:
   %cmpUp = icmp sgt i32 %x, 8388607
-  %cmpLow = icmp sgt i32 %x, -8388608
-  %saturateLow = select i1 %cmpLow, i32 -8388608, i32 %x
+  %0 = icmp slt i32 %x, -8388608
+  %saturateLow = select i1 %0, i32 %x, i32 -8388608
   %saturateUp = select i1 %cmpUp, i32 8388607, i32 %saturateLow
   ret i32 %saturateUp
 }
 
 ; x < k ? k : (x < -k ? -k : x)
 ; Second condition lower-saturates, first doesn't upper-saturate.
 define i32 @no_sat_missing_upper(i32 %x) #0 {
 ; CHECK-LABEL: no_sat_missing_upper:
 ; CHECK-NOT: ssat
 entry:
   %cmpUp = icmp slt i32 %x, 8388607
-  %cmpLow = icmp slt i32 %x, -8388608
-  %saturateLow = select i1 %cmpLow, i32 -8388608, i32 %x
+  %0 = icmp sgt i32 %x, -8388608
+  %saturateLow = select i1 %0, i32 %x, i32 -8388608
   %saturateUp = select i1 %cmpUp, i32 8388607, i32 %saturateLow
   ret i32 %saturateUp
 }
 
 ; Lower constant is different in the select and in the compare
 define i32 @no_sat_incorrect_constant(i32 %x) #0 {
 ; CHECK-LABEL: no_sat_incorrect_constant:
 ; CHECK-NOT: ssat
 entry:
   %cmpUp = icmp sgt i32 %x, 8388607
   %cmpLow = icmp slt i32 %x, -8388608
   %saturateLow = select i1 %cmpLow, i32 -8388607, i32 %x
   %saturateUp = select i1 %cmpUp, i32 8388607, i32 %saturateLow
   ret i32 %saturateUp
 }
 
 ; The interval is not [k, ~k]
 define i32 @no_sat_incorrect_interval(i32 %x) #0 {
 ; CHECK-LABEL: no_sat_incorrect_interval:
 ; CHECK-NOT: ssat
 entry:
-  %cmpUp = icmp sgt i32 %x, 8388607
-  %cmpLow = icmp slt i32 %x, -19088744
-  %saturateLow = select i1 %cmpLow, i32 -19088744, i32 %x
-  %saturateUp = select i1 %cmpUp, i32 8388607, i32 %saturateLow
+  %0 = icmp sgt i32 %x, -19088744
+  %saturateLow = select i1 %0, i32 %x, i32 -19088744
+  %1 = icmp slt i32 %saturateLow, 8388607
+  %saturateUp = select i1 %1, i32 %saturateLow, i32 8388607
   ret i32 %saturateUp
 }
 
 ; The returned value (y) is not the same as the tested value (x).
 define i32 @no_sat_incorrect_return(i32 %x, i32 %y) #0 {
 ; CHECK-LABEL: no_sat_incorrect_return:
 ; CHECK-NOT: ssat
 entry:

llvm/test/CodeGen/ARM/usat.ll

 ; x < 0 ? 0 : (x > k ? k : x)
 ; 32-bit base test
 define i32 @unsigned_sat_base_32bit(i32 %x) #0 {
 ; CHECK-LABEL: unsigned_sat_base_32bit:
 ; V6: usat r0, #23, r0
 ; V6T2: usat r0, #23, r0
 ; V4T-NOT: usat
 entry:
-  %cmpLow = icmp slt i32 %x, 0
-  %cmpUp = icmp sgt i32 %x, 8388607
-  %saturateUp = select i1 %cmpUp, i32 8388607, i32 %x
-  %saturateLow = select i1 %cmpLow, i32 0, i32 %saturateUp
+  %0 = icmp slt i32 %x, 8388607
+  %saturateUp = select i1 %0, i32 %x, i32 8388607
+  %1 = icmp sgt i32 %saturateUp, 0
+  %saturateLow = select i1 %1, i32 %saturateUp, i32 0
   ret i32 %saturateLow
 }
 
 ; x < 0 ? 0 : (x > k ? k : x)
 ; 16-bit base test
 define i16 @unsigned_sat_base_16bit(i16 %x) #0 {
 ; CHECK-LABEL: unsigned_sat_base_16bit:
 ; V6: usat r0, #11, r0
 ; V6T2: usat r0, #11, r0
 ; V4T-NOT: usat
 entry:
-  %cmpLow = icmp slt i16 %x, 0
-  %cmpUp = icmp sgt i16 %x, 2047
-  %saturateUp = select i1 %cmpUp, i16 2047, i16 %x
-  %saturateLow = select i1 %cmpLow, i16 0, i16 %saturateUp
+  %0 = icmp slt i16 %x, 2047
+  %saturateUp = select i1 %0, i16 %x, i16 2047
+  %1 = icmp sgt i16 %saturateUp, 0
+  %saturateLow = select i1 %1, i16 %saturateUp, i16 0
   ret i16 %saturateLow
 }
 
 ; x < 0 ? 0 : (x > k ? k : x)
 ; 8-bit base test
 define i8 @unsigned_sat_base_8bit(i8 %x) #0 {
 ; CHECK-LABEL: unsigned_sat_base_8bit:
 ; V6: usat r0, #5, r0
 ; V6T2: usat r0, #5, r0
 ; V4T-NOT: usat
 entry:
-  %cmpLow = icmp slt i8 %x, 0
-  %cmpUp = icmp sgt i8 %x, 31
-  %saturateUp = select i1 %cmpUp, i8 31, i8 %x
-  %saturateLow = select i1 %cmpLow, i8 0, i8 %saturateUp
+  %0 = icmp slt i8 %x, 31
+  %saturateUp = select i1 %0, i8 %x, i8 31
+  %1 = icmp sgt i8 %saturateUp, 0
+  %saturateLow = select i1 %1, i8 %saturateUp, i8 0
   ret i8 %saturateLow
 }
 
 ;
 ; Tests where the conditionals that check for upper and lower bounds,
 ; or the < and > operators, are arranged in different ways. Only some
 ; of the possible combinations that lead to USAT are tested.
 ;
 ; x < 0 ? 0 : (x < k ? x : k)
 define i32 @unsigned_sat_lower_upper_1(i32 %x) #0 {
 ; CHECK-LABEL: unsigned_sat_lower_upper_1:
 ; V6: usat r0, #23, r0
 ; V6T2: usat r0, #23, r0
 ; V4T-NOT: usat
 entry:
-  %cmpLow = icmp slt i32 %x, 0
   %cmpUp = icmp slt i32 %x, 8388607
   %saturateUp = select i1 %cmpUp, i32 %x, i32 8388607
-  %saturateLow = select i1 %cmpLow, i32 0, i32 %saturateUp
+  %0 = icmp sgt i32 %saturateUp, 0
+  %saturateLow = select i1 %0, i32 %saturateUp, i32 0
   ret i32 %saturateLow
 }
 
 ; x > 0 ? (x > k ? k : x) : 0
 define i32 @unsigned_sat_lower_upper_2(i32 %x) #0 {
 ; CHECK-LABEL: unsigned_sat_lower_upper_2:
 ; V6: usat    r0, #23, r0
 ; V6T2: usat    r0, #23, r0
 ; V4T-NOT: usat
 entry:
-  %cmpLow = icmp sgt i32 %x, 0
-  %cmpUp = icmp sgt i32 %x, 8388607
-  %saturateUp = select i1 %cmpUp, i32 8388607, i32 %x
-  %saturateLow = select i1 %cmpLow, i32 %saturateUp, i32 0
+  %0 = icmp slt i32 %x, 8388607
+  %saturateUp = select i1 %0, i32 %x, i32 8388607
+  %1 = icmp sgt i32 %saturateUp, 0
+  %saturateLow = select i1 %1, i32 %saturateUp, i32 0
   ret i32 %saturateLow
 }
 
 ; x < k ? (x < 0 ? 0 : x) : k
 define i32 @unsigned_sat_upper_lower_1(i32 %x) #0 {
 ; CHECK-LABEL: unsigned_sat_upper_lower_1:
 ; V6: usat    r0, #23, r0
 ; V6T2: usat    r0, #23, r0
 ; V4T-NOT: usat
 entry:
-  %cmpUp = icmp slt i32 %x, 8388607
-  %cmpLow = icmp slt i32 %x, 0
-  %saturateLow = select i1 %cmpLow, i32 0, i32 %x
-  %saturateUp = select i1 %cmpUp, i32 %saturateLow, i32 8388607
+  %0 = icmp sgt i32 %x, 0
+  %saturateLow = select i1 %0, i32 %x, i32 0
+  %1 = icmp slt i32 %saturateLow, 8388607
+  %saturateUp = select i1 %1, i32 %saturateLow, i32 8388607
   ret i32 %saturateUp
 }
 
 ; x > k ? k : (x < 0 ? 0 : x)
 define i32 @unsigned_sat_upper_lower_2(i32 %x) #0 {
 ; CHECK-LABEL: unsigned_sat_upper_lower_2:
 ; V6: usat    r0, #23, r0
 ; V6T2: usat    r0, #23, r0
 ; V4T-NOT: usat
 entry:
-  %cmpUp = icmp sgt i32 %x, 8388607
-  %cmpLow = icmp slt i32 %x, 0
-  %saturateLow = select i1 %cmpLow, i32 0, i32 %x
-  %saturateUp = select i1 %cmpUp, i32 8388607, i32 %saturateLow
+  %0 = icmp sgt i32 %x, 0
+  %saturateLow = select i1 %0, i32 %x, i32 0
+  %1 = icmp slt i32 %saturateLow, 8388607
+  %saturateUp = select i1 %1, i32 %saturateLow, i32 8388607
   ret i32 %saturateUp
 }
 
 ; k < x ? k : (x > 0 ? x : 0)
 define i32 @unsigned_sat_upper_lower_3(i32 %x) #0 {
 ; CHECK-LABEL: unsigned_sat_upper_lower_3:
 ; V6: usat    r0, #23, r0
 ; V6T2: usat    r0, #23, r0
 ; V4T-NOT: usat
 entry:
-  %cmpUp = icmp slt i32 8388607, %x
   %cmpLow = icmp sgt i32 %x, 0
   %saturateLow = select i1 %cmpLow, i32 %x, i32 0
-  %saturateUp = select i1 %cmpUp, i32 8388607, i32 %saturateLow
+  %0 = icmp slt i32 %saturateLow, 8388607
+  %saturateUp = select i1 %0, i32 %saturateLow, i32 8388607
   ret i32 %saturateUp
 }
 
 ;
 ; The following tests check for patterns that should not transform
 ; into USAT but are similar enough that could confuse the selector.
 ;
 ; x > k ? k : (x > 0 ? 0 : x)
 ; First condition upper-saturates, second doesn't lower-saturate.
 define i32 @no_unsigned_sat_missing_lower(i32 %x) #0 {
 ; CHECK-LABEL: no_unsigned_sat_missing_lower
 ; CHECK-NOT: usat
 entry:
   %cmpUp = icmp sgt i32 %x, 8388607
-  %cmpLow = icmp sgt i32 %x, 0
-  %saturateLow = select i1 %cmpLow, i32 0, i32 %x
+  %0 = icmp slt i32 %x, 0
+  %saturateLow = select i1 %0, i32 %x, i32 0
   %saturateUp = select i1 %cmpUp, i32 8388607, i32 %saturateLow
   ret i32 %saturateUp
 }
 
 ; x < k ? k : (x < 0 ? 0 : x)
 ; Second condition lower-saturates, first doesn't upper-saturate.
 define i32 @no_unsigned_sat_missing_upper(i32 %x) #0 {
 ; CHECK-LABEL: no_unsigned_sat_missing_upper:
 ; CHECK-NOT: usat
 entry:
   %cmpUp = icmp slt i32 %x, 8388607
-  %cmpLow = icmp slt i32 %x, 0
-  %saturateLow = select i1 %cmpLow, i32 0, i32 %x
+  %0 = icmp sgt i32 %x, 0
+  %saturateLow = select i1 %0, i32 %x, i32 0
   %saturateUp = select i1 %cmpUp, i32 8388607, i32 %saturateLow
   ret i32 %saturateUp
 }
 
 ; Lower constant is different in the select and in the compare
 define i32 @no_unsigned_sat_incorrect_constant(i32 %x) #0 {
 ; CHECK-LABEL: no_unsigned_sat_incorrect_constant:
 ; CHECK-NOT: usat
 entry:
-  %cmpUp = icmp sgt i32 %x, 8388607
-  %cmpLow = icmp slt i32 %x, 0
-  %saturateLow = select i1 %cmpLow, i32 -1, i32 %x
-  %saturateUp = select i1 %cmpUp, i32 8388607, i32 %saturateLow
+  %cmpLow.inv = icmp sgt i32 %x, -1
+  %saturateLow = select i1 %cmpLow.inv, i32 %x, i32 -1
+  %0 = icmp slt i32 %saturateLow, 8388607
+  %saturateUp = select i1 %0, i32 %saturateLow, i32 8388607
   ret i32 %saturateUp
 }
 
 ; The interval is not [0, k]
 define i32 @no_unsigned_sat_incorrect_interval(i32 %x) #0 {
 ; CHECK-LABEL: no_unsigned_sat_incorrect_interval:
 ; CHECK-NOT: usat
 entry:
-  %cmpUp = icmp sgt i32 %x, 8388607
-  %cmpLow = icmp slt i32 %x, -4
-  %saturateLow = select i1 %cmpLow, i32 -4, i32 %x
-  %saturateUp = select i1 %cmpUp, i32 8388607, i32 %saturateLow
+  %0 = icmp sgt i32 %x, -4
+  %saturateLow = select i1 %0, i32 %x, i32 -4
+  %1 = icmp slt i32 %saturateLow, 8388607
+  %saturateUp = select i1 %1, i32 %saturateLow, i32 8388607
   ret i32 %saturateUp
 }
 
 ; The returned value (y) is not the same as the tested value (x).
 define i32 @no_unsigned_sat_incorrect_return(i32 %x, i32 %y) #0 {
 ; CHECK-LABEL: no_unsigned_sat_incorrect_return:
 ; CHECK-NOT: usat
 entry:

llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll

 ; CHECK-NEXT:  .LBB13_6: @ %for.body.preheader21
 ; CHECK-NEXT:    sub.w lr, r3, r5
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB13_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrsb r0, [r12], #1
 ; CHECK-NEXT:    ldrsb r1, [r6], #1
 ; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    asrs r1, r0, #7
-; CHECK-NEXT:    cmn.w r1, #128
-; CHECK-NEXT:    mvn r1, #127
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    asrgt r1, r0, #7
-; CHECK-NEXT:    cmp r1, #127
-; CHECK-NEXT:    it ge
-; CHECK-NEXT:    movge r1, #127
-; CHECK-NEXT:    strb r1, [r4], #1
+; CHECK-NEXT:    asrs r0, r0, #7
+; CHECK-NEXT:    ssat r0, #8, r0
+; CHECK-NEXT:    strb r0, [r4], #1
 ; CHECK-NEXT:    le lr, .LBB13_7
 ; CHECK-NEXT:  .LBB13_8: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %cmp8 = icmp eq i32 %N, 0
   br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
 
 for.body.preheader:                               ; preds = %entry