[LegalizeTypes][AArch64][ARM] Teach PromoteIntegerResult to preserve NSW/NUW flags for ADD/SUB.

[craig.topper]

Summary

If the promoted inputs are known to be sign/zero extended, we can
preserve these flags when we promote the operation.

Alternatively, we could forcibly sign/zero extend the input in order
to preserve the flag, but that could lead to extra code that may
not be beneficial.

[efriedma]

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

1251

Do the flags on the original node actually matter here? Unless I'm missing something, the logic you're using applies whether or not the original node was marked nsw/nuw. (e.g. the sum of two zero-extended inputs is always nuw.)

[craig.topper]

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

1251

Good point. At best I guess its a compile time filter. Maybe this argues that this should be in DAGCombine? Or the avgfloor optimization in SimplifyDemandedBits could be made to work use computeKnownBits/ComputeNumSignBits instead of using nuw/nsw?

[nikic]

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

1251

It seems like combineShiftToAVG() already computes known bits / sign bits anyway, so might as well make use of them...

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
#endif
 
   case ISD::FP_TO_BF16:
   case ISD::FP_TO_FP16:
     Res = PromoteIntRes_FP_TO_FP16_BF16(N);
     break;
 
   case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break;
 
+  case ISD::ADD:
+  case ISD::SUB:
+  case ISD::VP_ADD:
+  case ISD::VP_SUB:       Res = PromoteIntRes_ADDSUB(N); break;
+
   case ISD::AND:
   case ISD::OR:
   case ISD::XOR:
-  case ISD::ADD:
-  case ISD::SUB:
   case ISD::MUL:
   case ISD::VP_AND:
   case ISD::VP_OR:
   case ISD::VP_XOR:
-  case ISD::VP_ADD:
-  case ISD::VP_SUB:
   case ISD::VP_MUL:      Res = PromoteIntRes_SimpleIntBinOp(N); break;
 
   case ISD::VP_SMIN:
   case ISD::VP_SMAX:
   case ISD::SDIV:
   case ISD::SREM:
   case ISD::VP_SDIV:
   case ISD::VP_SREM:     Res = PromoteIntRes_SExtIntBinOp(N); break;
SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
   if (N->getNumOperands() == 2)
     return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
   assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
   assert(N->isVPOpcode() && "Expected VP opcode");
   return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
                      N->getOperand(2), N->getOperand(3));
 }
 
+SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUB(SDNode *N) {
+  // The input may have strange things in the top bits of the registers, but
+  // these operations don't care.  They may have weird bits going out, but
+  // that too is okay if they are integer operations.
+  SDValue LHS = GetPromotedInteger(N->getOperand(0));
+  SDValue RHS = GetPromotedInteger(N->getOperand(1));
+  if (N->getNumOperands() == 2) {
+    SDNodeFlags Flags;
+    // Preserve nsw flag if the promoted inputs are sign extended.
+    if (N->getFlags().hasNoSignedWrap()) {
+      unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(LHS);
+      unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(RHS);
+      if (OpLEffectiveBits <= N->getOperand(0).getScalarValueSizeInBits() &&
+          OpREffectiveBits <= N->getOperand(1).getScalarValueSizeInBits())
+        Flags.setNoSignedWrap(true);
+    }
+
+    // Preserve nuw flag if the promoted inputs are zero extended.
+    if (N->getFlags().hasNoUnsignedWrap()) {
+      unsigned OpLEffectiveBits =
+          DAG.computeKnownBits(LHS).countMaxActiveBits();
+      unsigned OpREffectiveBits =
+          DAG.computeKnownBits(RHS).countMaxActiveBits();
+      if (OpLEffectiveBits <= N->getOperand(0).getScalarValueSizeInBits() &&
+          OpREffectiveBits <= N->getOperand(1).getScalarValueSizeInBits())
+        Flags.setNoUnsignedWrap(true);
+    }
+
+    return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
+                       Flags);
+  }
+
+  assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
+  assert(N->isVPOpcode() && "Expected VP opcode");
+  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
+                     N->getOperand(2), N->getOperand(3));
+}
+
 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
   // Sign extend the input.
   SDValue LHS = SExtPromotedInteger(N->getOperand(0));
   SDValue RHS = SExtPromotedInteger(N->getOperand(1));
   if (N->getNumOperands() == 2)
     return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
   assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
   assert(N->isVPOpcode() && "Expected VP opcode");

llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

 //===-- LegalizeTypes.h - DAG Type Legalizer class definition ---*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
private:
   SDValue PromoteIntRes_MGATHER(MaskedGatherSDNode *N);
   SDValue PromoteIntRes_Overflow(SDNode *N);
   SDValue PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo);
   SDValue PromoteIntRes_Select(SDNode *N);
   SDValue PromoteIntRes_SELECT_CC(SDNode *N);
   SDValue PromoteIntRes_SETCC(SDNode *N);
   SDValue PromoteIntRes_SHL(SDNode *N);
   SDValue PromoteIntRes_SimpleIntBinOp(SDNode *N);
+  SDValue PromoteIntRes_ADDSUB(SDNode *N);
   SDValue PromoteIntRes_ZExtIntBinOp(SDNode *N);
   SDValue PromoteIntRes_SExtIntBinOp(SDNode *N);
   SDValue PromoteIntRes_UMINUMAX(SDNode *N);
   SDValue PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N);
   SDValue PromoteIntRes_SRA(SDNode *N);
   SDValue PromoteIntRes_SRL(SDNode *N);
   SDValue PromoteIntRes_TRUNCATE(SDNode *N);
   SDValue PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo);

llvm/test/CodeGen/AArch64/arm64-vhadd.ll

 }
 
 define <2 x i16> @hadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
 ; CHECK-LABEL: hadd8x2_sext_asr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    shl.2s v0, v0, #24
 ; CHECK-NEXT:    shl.2s v1, v1, #24
 ; CHECK-NEXT:    sshr.2s v0, v0, #24
-; CHECK-NEXT:    ssra.2s v0, v1, #24
-; CHECK-NEXT:    sshr.2s v0, v0, #1
+; CHECK-NEXT:    sshr.2s v1, v1, #24
+; CHECK-NEXT:    shadd.2s v0, v0, v1
 ; CHECK-NEXT:    ret
   %zextsrc1 = sext <2 x i8> %src1 to <2 x i16>
   %zextsrc2 = sext <2 x i8> %src2 to <2 x i16>
   %add = add nsw <2 x i16> %zextsrc1, %zextsrc2
   %resulti8 = ashr <2 x i16> %add, <i16 1, i16 1>
   ret <2 x i16> %resulti8
 }
 
 define <2 x i16> @hadd8x2_zext_asr(<2 x i8> %src1, <2 x i8> %src2) {
 ; CHECK-LABEL: hadd8x2_zext_asr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi d2, #0x0000ff000000ff
 ; CHECK-NEXT:    and.8b v0, v0, v2
 ; CHECK-NEXT:    and.8b v1, v1, v2
-; CHECK-NEXT:    add.2s v0, v0, v1
-; CHECK-NEXT:    ushr.2s v0, v0, #1
+; CHECK-NEXT:    uhadd.2s v0, v0, v1
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
   %zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
   %add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
   %resulti8 = lshr <2 x i16> %add, <i16 1, i16 1>
   ret <2 x i16> %resulti8
 }
 

 }
 
 define <2 x i16> @hadd8x2_zext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
 ; CHECK-LABEL: hadd8x2_zext_lsr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi d2, #0x0000ff000000ff
 ; CHECK-NEXT:    and.8b v0, v0, v2
 ; CHECK-NEXT:    and.8b v1, v1, v2
-; CHECK-NEXT:    add.2s v0, v0, v1
-; CHECK-NEXT:    ushr.2s v0, v0, #1
+; CHECK-NEXT:    uhadd.2s v0, v0, v1
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
   %zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
   %add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
   %resulti8 = lshr <2 x i16> %add, <i16 1, i16 1>
   ret <2 x i16> %resulti8
 }
 

 
 define <2 x i16> @rhadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
 ; CHECK-LABEL: rhadd8x2_sext_asr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    shl.2s v0, v0, #24
 ; CHECK-NEXT:    shl.2s v1, v1, #24
 ; CHECK-NEXT:    sshr.2s v0, v0, #24
 ; CHECK-NEXT:    sshr.2s v1, v1, #24
-; CHECK-NEXT:    mvn.8b v0, v0
-; CHECK-NEXT:    sub.2s v0, v1, v0
-; CHECK-NEXT:    sshr.2s v0, v0, #1
+; CHECK-NEXT:    srhadd.2s v0, v0, v1
 ; CHECK-NEXT:    ret
   %zextsrc1 = sext <2 x i8> %src1 to <2 x i16>
   %zextsrc2 = sext <2 x i8> %src2 to <2 x i16>
   %add = add nsw <2 x i16> %zextsrc1, %zextsrc2
   %add2 = add nsw <2 x i16> %add, <i16 1, i16 1>
   %resulti8 = ashr <2 x i16> %add2, <i16 1, i16 1>
   ret <2 x i16> %resulti8
 }
 
 define <2 x i16> @rhadd8x2_zext_asr(<2 x i8> %src1, <2 x i8> %src2) {
 ; CHECK-LABEL: rhadd8x2_zext_asr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi d2, #0x0000ff000000ff
 ; CHECK-NEXT:    and.8b v0, v0, v2
 ; CHECK-NEXT:    and.8b v1, v1, v2
-; CHECK-NEXT:    mvn.8b v0, v0
-; CHECK-NEXT:    sub.2s v0, v1, v0
-; CHECK-NEXT:    ushr.2s v0, v0, #1
+; CHECK-NEXT:    urhadd.2s v0, v0, v1
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
   %zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
   %add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
   %add2 = add nuw nsw <2 x i16> %add, <i16 1, i16 1>
   %resulti8 = lshr <2 x i16> %add2, <i16 1, i16 1>
   ret <2 x i16> %resulti8
 }
 
 define <2 x i16> @rhadd8x2_sext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
 ; CHECK-LABEL: rhadd8x2_sext_lsr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    shl.2s v0, v0, #24
 ; CHECK-NEXT:    shl.2s v1, v1, #24
-; CHECK-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-NEXT:    movi.2s v2, #1
 ; CHECK-NEXT:    sshr.2s v0, v0, #24
-; CHECK-NEXT:    sshr.2s v1, v1, #24
-; CHECK-NEXT:    mvn.8b v0, v0
-; CHECK-NEXT:    sub.2s v0, v1, v0
-; CHECK-NEXT:    and.8b v0, v0, v2
+; CHECK-NEXT:    ssra.2s v0, v1, #24
+; CHECK-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-NEXT:    add.2s v0, v0, v2
+; CHECK-NEXT:    and.8b v0, v0, v1
 ; CHECK-NEXT:    ushr.2s v0, v0, #1
 ; CHECK-NEXT:    ret
   %zextsrc1 = sext <2 x i8> %src1 to <2 x i16>
   %zextsrc2 = sext <2 x i8> %src2 to <2 x i16>
   %add = add nsw <2 x i16> %zextsrc1, %zextsrc2
   %add2 = add nsw <2 x i16> %add, <i16 1, i16 1>
   %resulti8 = lshr <2 x i16> %add2, <i16 1, i16 1>
   ret <2 x i16> %resulti8
 }
 
 define <2 x i16> @rhadd8x2_zext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
 ; CHECK-LABEL: rhadd8x2_zext_lsr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi d2, #0x0000ff000000ff
 ; CHECK-NEXT:    and.8b v0, v0, v2
 ; CHECK-NEXT:    and.8b v1, v1, v2
-; CHECK-NEXT:    mvn.8b v0, v0
-; CHECK-NEXT:    sub.2s v0, v1, v0
-; CHECK-NEXT:    ushr.2s v0, v0, #1
+; CHECK-NEXT:    urhadd.2s v0, v0, v1
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
   %zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
   %add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
   %add2 = add nuw nsw <2 x i16> %add, <i16 1, i16 1>
   %resulti8 = lshr <2 x i16> %add2, <i16 1, i16 1>
   ret <2 x i16> %resulti8
 }

llvm/test/CodeGen/AArch64/sve-hadd.ll

entry:
   %s1s = zext <vscale x 4 x i32> %s1 to <vscale x 4 x i64>
   %m = add nuw nsw <vscale x 4 x i64> %s0s, %s1s
   %s = lshr <vscale x 4 x i64> %m, shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
   %s2 = trunc <vscale x 4 x i64> %s to <vscale x 4 x i32>
   ret <vscale x 4 x i32> %s2
 }
 
 define <vscale x 2 x i16> @hadds_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {
-; CHECK-LABEL: hadds_v2i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    sxth z0.d, p0/m, z0.d
-; CHECK-NEXT:    sxth z1.d, p0/m, z1.d
-; CHECK-NEXT:    add z0.d, z0.d, z1.d
-; CHECK-NEXT:    asr z0.d, z0.d, #1
-; CHECK-NEXT:    ret
+; SVE-LABEL: hadds_v2i16:
+; SVE:       // %bb.0: // %entry
+; SVE-NEXT:    ptrue p0.d
+; SVE-NEXT:    sxth z0.d, p0/m, z0.d
+; SVE-NEXT:    sxth z1.d, p0/m, z1.d
+; SVE-NEXT:    add z0.d, z0.d, z1.d
+; SVE-NEXT:    asr z0.d, z0.d, #1
+; SVE-NEXT:    ret
+;
+; SVE2-LABEL: hadds_v2i16:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    ptrue p0.d
+; SVE2-NEXT:    sxth z0.d, p0/m, z0.d
+; SVE2-NEXT:    sxth z1.d, p0/m, z1.d
+; SVE2-NEXT:    shadd z0.d, p0/m, z0.d, z1.d
+; SVE2-NEXT:    ret
 entry:
   %s0s = sext <vscale x 2 x i16> %s0 to <vscale x 2 x i32>
   %s1s = sext <vscale x 2 x i16> %s1 to <vscale x 2 x i32>
   %m = add nsw <vscale x 2 x i32> %s0s, %s1s
   %s = ashr <vscale x 2 x i32> %m, shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
   %s2 = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>
   ret <vscale x 2 x i16> %s2
 }
entry:
   %s1s = sext <vscale x 2 x i16> %s1 to <vscale x 2 x i32>
   %m = add nsw <vscale x 2 x i32> %s0s, %s1s
   %s = lshr <vscale x 2 x i32> %m, shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
   %s2 = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>
   ret <vscale x 2 x i16> %s2
 }
 
 define <vscale x 2 x i16> @haddu_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {
-; CHECK-LABEL: haddu_v2i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    and z0.d, z0.d, #0xffff
-; CHECK-NEXT:    and z1.d, z1.d, #0xffff
-; CHECK-NEXT:    add z0.d, z0.d, z1.d
-; CHECK-NEXT:    lsr z0.d, z0.d, #1
-; CHECK-NEXT:    ret
+; SVE-LABEL: haddu_v2i16:
+; SVE:       // %bb.0: // %entry
+; SVE-NEXT:    and z0.d, z0.d, #0xffff
+; SVE-NEXT:    and z1.d, z1.d, #0xffff
+; SVE-NEXT:    add z0.d, z0.d, z1.d
+; SVE-NEXT:    lsr z0.d, z0.d, #1
+; SVE-NEXT:    ret
+;
+; SVE2-LABEL: haddu_v2i16:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    ptrue p0.d
+; SVE2-NEXT:    and z0.d, z0.d, #0xffff
+; SVE2-NEXT:    and z1.d, z1.d, #0xffff
+; SVE2-NEXT:    uhadd z0.d, p0/m, z0.d, z1.d
+; SVE2-NEXT:    ret
 entry:
   %s0s = zext <vscale x 2 x i16> %s0 to <vscale x 2 x i32>
   %s1s = zext <vscale x 2 x i16> %s1 to <vscale x 2 x i32>
   %m = add nuw nsw <vscale x 2 x i32> %s0s, %s1s
   %s = lshr <vscale x 2 x i32> %m, shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
   %s2 = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>
   ret <vscale x 2 x i16> %s2
 }
entry:
   %s1s = zext <vscale x 8 x i16> %s1 to <vscale x 8 x i32>
   %m = add nuw nsw <vscale x 8 x i32> %s0s, %s1s
   %s = lshr <vscale x 8 x i32> %m, shufflevector (<vscale x 8 x i32> insertelement (<vscale x 8 x i32> poison, i32 1, i32 0), <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer)
   %s2 = trunc <vscale x 8 x i32> %s to <vscale x 8 x i16>
   ret <vscale x 8 x i16> %s2
 }
 
 define <vscale x 4 x i8> @hadds_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {
-; CHECK-LABEL: hadds_v4i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    sxtb z0.s, p0/m, z0.s
-; CHECK-NEXT:    sxtb z1.s, p0/m, z1.s
-; CHECK-NEXT:    add z0.s, z0.s, z1.s
-; CHECK-NEXT:    asr z0.s, z0.s, #1
-; CHECK-NEXT:    ret
+; SVE-LABEL: hadds_v4i8:
+; SVE:       // %bb.0: // %entry
+; SVE-NEXT:    ptrue p0.s
+; SVE-NEXT:    sxtb z0.s, p0/m, z0.s
+; SVE-NEXT:    sxtb z1.s, p0/m, z1.s
+; SVE-NEXT:    add z0.s, z0.s, z1.s
+; SVE-NEXT:    asr z0.s, z0.s, #1
+; SVE-NEXT:    ret
+;
+; SVE2-LABEL: hadds_v4i8:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    ptrue p0.s
+; SVE2-NEXT:    sxtb z0.s, p0/m, z0.s
+; SVE2-NEXT:    sxtb z1.s, p0/m, z1.s
+; SVE2-NEXT:    shadd z0.s, p0/m, z0.s, z1.s
+; SVE2-NEXT:    ret
 entry:
   %s0s = sext <vscale x 4 x i8> %s0 to <vscale x 4 x i16>
   %s1s = sext <vscale x 4 x i8> %s1 to <vscale x 4 x i16>
   %m = add nsw <vscale x 4 x i16> %s0s, %s1s
   %s = ashr <vscale x 4 x i16> %m, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 1, i32 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
   %s2 = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>
   ret <vscale x 4 x i8> %s2
 }
entry:
   %s1s = sext <vscale x 4 x i8> %s1 to <vscale x 4 x i16>
   %m = add nsw <vscale x 4 x i16> %s0s, %s1s
   %s = lshr <vscale x 4 x i16> %m, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 1, i32 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
   %s2 = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>
   ret <vscale x 4 x i8> %s2
 }
 
 define <vscale x 4 x i8> @haddu_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {
-; CHECK-LABEL: haddu_v4i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    and z0.s, z0.s, #0xff
-; CHECK-NEXT:    and z1.s, z1.s, #0xff
-; CHECK-NEXT:    add z0.s, z0.s, z1.s
-; CHECK-NEXT:    lsr z0.s, z0.s, #1
-; CHECK-NEXT:    ret
+; SVE-LABEL: haddu_v4i8:
+; SVE:       // %bb.0: // %entry
+; SVE-NEXT:    and z0.s, z0.s, #0xff
+; SVE-NEXT:    and z1.s, z1.s, #0xff
+; SVE-NEXT:    add z0.s, z0.s, z1.s
+; SVE-NEXT:    lsr z0.s, z0.s, #1
+; SVE-NEXT:    ret
+;
+; SVE2-LABEL: haddu_v4i8:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    ptrue p0.s
+; SVE2-NEXT:    and z0.s, z0.s, #0xff
+; SVE2-NEXT:    and z1.s, z1.s, #0xff
+; SVE2-NEXT:    uhadd z0.s, p0/m, z0.s, z1.s
+; SVE2-NEXT:    ret
 entry:
   %s0s = zext <vscale x 4 x i8> %s0 to <vscale x 4 x i16>
   %s1s = zext <vscale x 4 x i8> %s1 to <vscale x 4 x i16>
   %m = add nuw nsw <vscale x 4 x i16> %s0s, %s1s
   %s = lshr <vscale x 4 x i16> %m, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 1, i32 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
   %s2 = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>
   ret <vscale x 4 x i8> %s2
 }
entry:
   %add = add <vscale x 2 x i32> %s0s, shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
   %add2 = add <vscale x 2 x i32> %add, %s1s
   %s = lshr <vscale x 2 x i32> %add2, shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
   %result = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>
   ret <vscale x 2 x i16> %result
 }
 
 define <vscale x 2 x i16> @rhaddu_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {
-; CHECK-LABEL: rhaddu_v2i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov z2.d, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    and z0.d, z0.d, #0xffff
-; CHECK-NEXT:    and z1.d, z1.d, #0xffff
-; CHECK-NEXT:    eor z0.d, z0.d, z2.d
-; CHECK-NEXT:    sub z0.d, z1.d, z0.d
-; CHECK-NEXT:    lsr z0.d, z0.d, #1
-; CHECK-NEXT:    ret
+; SVE-LABEL: rhaddu_v2i16:
+; SVE:       // %bb.0: // %entry
+; SVE-NEXT:    mov z2.d, #-1 // =0xffffffffffffffff
+; SVE-NEXT:    and z0.d, z0.d, #0xffff
+; SVE-NEXT:    and z1.d, z1.d, #0xffff
+; SVE-NEXT:    eor z0.d, z0.d, z2.d
+; SVE-NEXT:    sub z0.d, z1.d, z0.d
+; SVE-NEXT:    lsr z0.d, z0.d, #1
+; SVE-NEXT:    ret
+;
+; SVE2-LABEL: rhaddu_v2i16:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    ptrue p0.d
+; SVE2-NEXT:    and z0.d, z0.d, #0xffff
+; SVE2-NEXT:    and z1.d, z1.d, #0xffff
+; SVE2-NEXT:    urhadd z0.d, p0/m, z0.d, z1.d
+; SVE2-NEXT:    ret
 entry:
   %s0s = zext <vscale x 2 x i16> %s0 to <vscale x 2 x i32>
   %s1s = zext <vscale x 2 x i16> %s1 to <vscale x 2 x i32>
   %add = add nuw nsw <vscale x 2 x i32> %s0s, shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
   %add2 = add nuw nsw <vscale x 2 x i32> %add, %s1s
   %s = lshr <vscale x 2 x i32> %add2, shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
   %result = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>
   ret <vscale x 2 x i16> %result
entry:
   %add = add <vscale x 4 x i16> %s0s, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 1, i32 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
   %add2 = add <vscale x 4 x i16> %add, %s1s
   %s = lshr <vscale x 4 x i16> %add2, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 1, i32 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
   %result = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>
   ret <vscale x 4 x i8> %result
 }
 
 define <vscale x 4 x i8> @rhaddu_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {
-; CHECK-LABEL: rhaddu_v4i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov z2.s, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    and z0.s, z0.s, #0xff
-; CHECK-NEXT:    and z1.s, z1.s, #0xff
-; CHECK-NEXT:    eor z0.d, z0.d, z2.d
-; CHECK-NEXT:    sub z0.s, z1.s, z0.s
-; CHECK-NEXT:    lsr z0.s, z0.s, #1
-; CHECK-NEXT:    ret
+; SVE-LABEL: rhaddu_v4i8:
+; SVE:       // %bb.0: // %entry
+; SVE-NEXT:    mov z2.s, #-1 // =0xffffffffffffffff
+; SVE-NEXT:    and z0.s, z0.s, #0xff
+; SVE-NEXT:    and z1.s, z1.s, #0xff
+; SVE-NEXT:    eor z0.d, z0.d, z2.d
+; SVE-NEXT:    sub z0.s, z1.s, z0.s
+; SVE-NEXT:    lsr z0.s, z0.s, #1
+; SVE-NEXT:    ret
+;
+; SVE2-LABEL: rhaddu_v4i8:
+; SVE2:       // %bb.0: // %entry
+; SVE2-NEXT:    ptrue p0.s
+; SVE2-NEXT:    and z0.s, z0.s, #0xff
+; SVE2-NEXT:    and z1.s, z1.s, #0xff
+; SVE2-NEXT:    urhadd z0.s, p0/m, z0.s, z1.s
+; SVE2-NEXT:    ret
 entry:
   %s0s = zext <vscale x 4 x i8> %s0 to <vscale x 4 x i16>
   %s1s = zext <vscale x 4 x i8> %s1 to <vscale x 4 x i16>
   %add = add nuw nsw <vscale x 4 x i16> %s0s, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 1, i32 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
   %add2 = add nuw nsw <vscale x 4 x i16> %add, %s1s
   %s = lshr <vscale x 4 x i16> %add2, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 1, i32 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
   %result = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>
   ret <vscale x 4 x i8> %result

llvm/test/CodeGen/Thumb2/mve-vhadd.ll

 }
 
 define arm_aapcs_vfpcc <4 x i8> @vhaddu_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
 ; CHECK-LABEL: vhaddu_v4i8:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmov.i32 q2, #0xff
 ; CHECK-NEXT:    vand q1, q1, q2
 ; CHECK-NEXT:    vand q0, q0, q2
-; CHECK-NEXT:    vadd.i32 q0, q0, q1
-; CHECK-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-NEXT:    vhadd.u32 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = zext <4 x i8> %s0 to <4 x i16>
   %s1s = zext <4 x i8> %s1 to <4 x i16>
   %m = add nuw nsw <4 x i16> %s0s, %s1s
   %s = lshr <4 x i16> %m, <i16 1, i16 1, i16 1, i16 1>
   %s2 = trunc <4 x i16> %s to <4 x i8>
   ret <4 x i8> %s2
entry:
   %result = trunc <4 x i16> %s to <4 x i8>
   ret <4 x i8> %result
 }
 
 define arm_aapcs_vfpcc <4 x i8> @vrhaddu_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
 ; CHECK-LABEL: vrhaddu_v4i8:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmov.i32 q2, #0xff
-; CHECK-NEXT:    movs r0, #1
 ; CHECK-NEXT:    vand q1, q1, q2
 ; CHECK-NEXT:    vand q0, q0, q2
-; CHECK-NEXT:    vadd.i32 q0, q0, q1
-; CHECK-NEXT:    vadd.i32 q0, q0, r0
-; CHECK-NEXT:    vshr.u32 q0, q0, #1
+; CHECK-NEXT:    vrhadd.u32 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = zext <4 x i8> %s0 to <4 x i16>
   %s1s = zext <4 x i8> %s1 to <4 x i16>
   %add = add nuw nsw <4 x i16> %s0s, <i16 1, i16 1, i16 1, i16 1>
   %add2 = add nuw nsw <4 x i16> %add, %s1s
   %s = lshr <4 x i16> %add2, <i16 1, i16 1, i16 1, i16 1>
   %result = trunc <4 x i16> %s to <4 x i8>