[SelectionDAG][AArch64][ARM] Remove setFlags call from DAGTypeLegalizer::SetPromotedInteger.

[craig.topper]

Summary

This was originally added to preserve FMF on SETCC. Unfortunately,
it also incorrectly preserves nuw/nsw on ADD/SUB in some cases.

There's also no guarantee the new opcode is even the same opcode
as the original node.

This patch removes the code and adds code to explicitly preserve
FMF flags in the SETCC promotion function.

The other test changes are from nuw/nsw not being preserved. I
believe for all these tests it was correct to preserve the flags,
so we need new code to preserve the flags when possible. I'll post
another patch for that since it's a riskier change.

This should unblock D150769.

[efriedma]

LGTM

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
 
   SDLoc dl(N);
   assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
          "Vector compare must return a vector result!");
 
   // Get the SETCC result using the canonical SETCC type.
   SDValue SetCC;
   if (N->isStrictFPOpcode()) {
-    EVT VTs[] = {SVT, MVT::Other};
+    SDVTList VTs = DAG.getVTList({SVT, MVT::Other});
     SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
                        N->getOperand(2), N->getOperand(3)};
-    SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers);
+    SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers, N->getFlags());
     // Legalize the chain result - switch anything that used the old chain to
     // use the new one.
     ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
   } else
     SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
-                        N->getOperand(1), N->getOperand(2));
+                        N->getOperand(1), N->getOperand(2), N->getFlags());
 
   // Convert to the expected type.
   return DAG.getSExtOrTrunc(SetCC, dl, NVT);
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) {
   SDLoc DL(N);
   SDValue Arg = N->getOperand(0);

llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp

 //===-- LegalizeTypes.cpp - Common code for DAG type legalizer ------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
void DAGTypeLegalizer::SetPromotedInteger(SDValue Op, SDValue Result) {
   assert(Result.getValueType() ==
          TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
          "Invalid type for promoted integer");
   AnalyzeNewValue(Result);
 
   auto &OpIdEntry = PromotedIntegers[getTableId(Op)];
   assert((OpIdEntry == 0) && "Node is already promoted!");
   OpIdEntry = getTableId(Result);
-  Result->setFlags(Op->getFlags());
 
   DAG.transferDbgValues(Op, Result);
 }
 
 void DAGTypeLegalizer::SetSoftenedFloat(SDValue Op, SDValue Result) {
 #ifndef NDEBUG
   EVT VT = Result.getValueType();
   LLVMContext &Ctx = *DAG.getContext();

llvm/test/CodeGen/AArch64/arm64-vhadd.ll

 }
 
 define <2 x i16> @hadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
 ; CHECK-LABEL: hadd8x2_sext_asr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    shl.2s v0, v0, #24
 ; CHECK-NEXT:    shl.2s v1, v1, #24
 ; CHECK-NEXT:    sshr.2s v0, v0, #24
-; CHECK-NEXT:    sshr.2s v1, v1, #24
-; CHECK-NEXT:    shadd.2s v0, v0, v1
+; CHECK-NEXT:    ssra.2s v0, v1, #24
+; CHECK-NEXT:    sshr.2s v0, v0, #1
 ; CHECK-NEXT:    ret
   %zextsrc1 = sext <2 x i8> %src1 to <2 x i16>
   %zextsrc2 = sext <2 x i8> %src2 to <2 x i16>
   %add = add nsw <2 x i16> %zextsrc1, %zextsrc2
   %resulti8 = ashr <2 x i16> %add, <i16 1, i16 1>
   ret <2 x i16> %resulti8
 }
 
 define <2 x i16> @hadd8x2_zext_asr(<2 x i8> %src1, <2 x i8> %src2) {
 ; CHECK-LABEL: hadd8x2_zext_asr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi d2, #0x0000ff000000ff
 ; CHECK-NEXT:    and.8b v0, v0, v2
 ; CHECK-NEXT:    and.8b v1, v1, v2
-; CHECK-NEXT:    uhadd.2s v0, v0, v1
+; CHECK-NEXT:    add.2s v0, v0, v1
+; CHECK-NEXT:    ushr.2s v0, v0, #1
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
   %zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
   %add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
   %resulti8 = lshr <2 x i16> %add, <i16 1, i16 1>
   ret <2 x i16> %resulti8
 }
 

 }
 
 define <2 x i16> @hadd8x2_zext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
 ; CHECK-LABEL: hadd8x2_zext_lsr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi d2, #0x0000ff000000ff
 ; CHECK-NEXT:    and.8b v0, v0, v2
 ; CHECK-NEXT:    and.8b v1, v1, v2
-; CHECK-NEXT:    uhadd.2s v0, v0, v1
+; CHECK-NEXT:    add.2s v0, v0, v1
+; CHECK-NEXT:    ushr.2s v0, v0, #1
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
   %zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
   %add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
   %resulti8 = lshr <2 x i16> %add, <i16 1, i16 1>
   ret <2 x i16> %resulti8
 }
 

 
 define <2 x i16> @rhadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {
 ; CHECK-LABEL: rhadd8x2_sext_asr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    shl.2s v0, v0, #24
 ; CHECK-NEXT:    shl.2s v1, v1, #24
 ; CHECK-NEXT:    sshr.2s v0, v0, #24
 ; CHECK-NEXT:    sshr.2s v1, v1, #24
-; CHECK-NEXT:    srhadd.2s v0, v0, v1
+; CHECK-NEXT:    mvn.8b v0, v0
+; CHECK-NEXT:    sub.2s v0, v1, v0
+; CHECK-NEXT:    sshr.2s v0, v0, #1
 ; CHECK-NEXT:    ret
   %zextsrc1 = sext <2 x i8> %src1 to <2 x i16>
   %zextsrc2 = sext <2 x i8> %src2 to <2 x i16>
   %add = add nsw <2 x i16> %zextsrc1, %zextsrc2
   %add2 = add nsw <2 x i16> %add, <i16 1, i16 1>
   %resulti8 = ashr <2 x i16> %add2, <i16 1, i16 1>
   ret <2 x i16> %resulti8
 }
 
 define <2 x i16> @rhadd8x2_zext_asr(<2 x i8> %src1, <2 x i8> %src2) {
 ; CHECK-LABEL: rhadd8x2_zext_asr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi d2, #0x0000ff000000ff
 ; CHECK-NEXT:    and.8b v0, v0, v2
 ; CHECK-NEXT:    and.8b v1, v1, v2
-; CHECK-NEXT:    urhadd.2s v0, v0, v1
+; CHECK-NEXT:    mvn.8b v0, v0
+; CHECK-NEXT:    sub.2s v0, v1, v0
+; CHECK-NEXT:    ushr.2s v0, v0, #1
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
   %zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
   %add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
   %add2 = add nuw nsw <2 x i16> %add, <i16 1, i16 1>
   %resulti8 = lshr <2 x i16> %add2, <i16 1, i16 1>
   ret <2 x i16> %resulti8
 }
 
 define <2 x i16> @rhadd8x2_sext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
 ; CHECK-LABEL: rhadd8x2_sext_lsr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    shl.2s v0, v0, #24
 ; CHECK-NEXT:    shl.2s v1, v1, #24
-; CHECK-NEXT:    movi.2s v2, #1
+; CHECK-NEXT:    movi d2, #0x00ffff0000ffff
 ; CHECK-NEXT:    sshr.2s v0, v0, #24
-; CHECK-NEXT:    ssra.2s v0, v1, #24
-; CHECK-NEXT:    movi d1, #0x00ffff0000ffff
-; CHECK-NEXT:    add.2s v0, v0, v2
-; CHECK-NEXT:    and.8b v0, v0, v1
+; CHECK-NEXT:    sshr.2s v1, v1, #24
+; CHECK-NEXT:    mvn.8b v0, v0
+; CHECK-NEXT:    sub.2s v0, v1, v0
+; CHECK-NEXT:    and.8b v0, v0, v2
 ; CHECK-NEXT:    ushr.2s v0, v0, #1
 ; CHECK-NEXT:    ret
   %zextsrc1 = sext <2 x i8> %src1 to <2 x i16>
   %zextsrc2 = sext <2 x i8> %src2 to <2 x i16>
   %add = add nsw <2 x i16> %zextsrc1, %zextsrc2
   %add2 = add nsw <2 x i16> %add, <i16 1, i16 1>
   %resulti8 = lshr <2 x i16> %add2, <i16 1, i16 1>
   ret <2 x i16> %resulti8
 }
 
 define <2 x i16> @rhadd8x2_zext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
 ; CHECK-LABEL: rhadd8x2_zext_lsr:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi d2, #0x0000ff000000ff
 ; CHECK-NEXT:    and.8b v0, v0, v2
 ; CHECK-NEXT:    and.8b v1, v1, v2
-; CHECK-NEXT:    urhadd.2s v0, v0, v1
+; CHECK-NEXT:    mvn.8b v0, v0
+; CHECK-NEXT:    sub.2s v0, v1, v0
+; CHECK-NEXT:    ushr.2s v0, v0, #1
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <2 x i8> %src1 to <2 x i16>
   %zextsrc2 = zext <2 x i8> %src2 to <2 x i16>
   %add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc2
   %add2 = add nuw nsw <2 x i16> %add, <i16 1, i16 1>
   %resulti8 = lshr <2 x i16> %add2, <i16 1, i16 1>
   ret <2 x i16> %resulti8
 }

llvm/test/CodeGen/AArch64/sve-hadd.ll

entry:
   %s1s = zext <vscale x 4 x i32> %s1 to <vscale x 4 x i64>
   %m = add nuw nsw <vscale x 4 x i64> %s0s, %s1s
   %s = lshr <vscale x 4 x i64> %m, shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
   %s2 = trunc <vscale x 4 x i64> %s to <vscale x 4 x i32>
   ret <vscale x 4 x i32> %s2
 }
 
 define <vscale x 2 x i16> @hadds_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {
-; SVE-LABEL: hadds_v2i16:
-; SVE:       // %bb.0: // %entry
-; SVE-NEXT:    ptrue p0.d
-; SVE-NEXT:    sxth z0.d, p0/m, z0.d
-; SVE-NEXT:    sxth z1.d, p0/m, z1.d
-; SVE-NEXT:    add z0.d, z0.d, z1.d
-; SVE-NEXT:    asr z0.d, z0.d, #1
-; SVE-NEXT:    ret
-;
-; SVE2-LABEL: hadds_v2i16:
-; SVE2:       // %bb.0: // %entry
-; SVE2-NEXT:    ptrue p0.d
-; SVE2-NEXT:    sxth z0.d, p0/m, z0.d
-; SVE2-NEXT:    sxth z1.d, p0/m, z1.d
-; SVE2-NEXT:    shadd z0.d, p0/m, z0.d, z1.d
-; SVE2-NEXT:    ret
+; CHECK-LABEL: hadds_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    sxth z0.d, p0/m, z0.d
+; CHECK-NEXT:    sxth z1.d, p0/m, z1.d
+; CHECK-NEXT:    add z0.d, z0.d, z1.d
+; CHECK-NEXT:    asr z0.d, z0.d, #1
+; CHECK-NEXT:    ret
 entry:
   %s0s = sext <vscale x 2 x i16> %s0 to <vscale x 2 x i32>
   %s1s = sext <vscale x 2 x i16> %s1 to <vscale x 2 x i32>
   %m = add nsw <vscale x 2 x i32> %s0s, %s1s
   %s = ashr <vscale x 2 x i32> %m, shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
   %s2 = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>
   ret <vscale x 2 x i16> %s2
 }
entry:
   %s1s = sext <vscale x 2 x i16> %s1 to <vscale x 2 x i32>
   %m = add nsw <vscale x 2 x i32> %s0s, %s1s
   %s = lshr <vscale x 2 x i32> %m, shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
   %s2 = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>
   ret <vscale x 2 x i16> %s2
 }
 
 define <vscale x 2 x i16> @haddu_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {
-; SVE-LABEL: haddu_v2i16:
-; SVE:       // %bb.0: // %entry
-; SVE-NEXT:    and z0.d, z0.d, #0xffff
-; SVE-NEXT:    and z1.d, z1.d, #0xffff
-; SVE-NEXT:    add z0.d, z0.d, z1.d
-; SVE-NEXT:    lsr z0.d, z0.d, #1
-; SVE-NEXT:    ret
-;
-; SVE2-LABEL: haddu_v2i16:
-; SVE2:       // %bb.0: // %entry
-; SVE2-NEXT:    ptrue p0.d
-; SVE2-NEXT:    and z0.d, z0.d, #0xffff
-; SVE2-NEXT:    and z1.d, z1.d, #0xffff
-; SVE2-NEXT:    uhadd z0.d, p0/m, z0.d, z1.d
-; SVE2-NEXT:    ret
+; CHECK-LABEL: haddu_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and z0.d, z0.d, #0xffff
+; CHECK-NEXT:    and z1.d, z1.d, #0xffff
+; CHECK-NEXT:    add z0.d, z0.d, z1.d
+; CHECK-NEXT:    lsr z0.d, z0.d, #1
+; CHECK-NEXT:    ret
 entry:
   %s0s = zext <vscale x 2 x i16> %s0 to <vscale x 2 x i32>
   %s1s = zext <vscale x 2 x i16> %s1 to <vscale x 2 x i32>
   %m = add nuw nsw <vscale x 2 x i32> %s0s, %s1s
   %s = lshr <vscale x 2 x i32> %m, shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
   %s2 = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>
   ret <vscale x 2 x i16> %s2
 }
entry:
   %s1s = zext <vscale x 8 x i16> %s1 to <vscale x 8 x i32>
   %m = add nuw nsw <vscale x 8 x i32> %s0s, %s1s
   %s = lshr <vscale x 8 x i32> %m, shufflevector (<vscale x 8 x i32> insertelement (<vscale x 8 x i32> poison, i32 1, i32 0), <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer)
   %s2 = trunc <vscale x 8 x i32> %s to <vscale x 8 x i16>
   ret <vscale x 8 x i16> %s2
 }
 
 define <vscale x 4 x i8> @hadds_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {
-; SVE-LABEL: hadds_v4i8:
-; SVE:       // %bb.0: // %entry
-; SVE-NEXT:    ptrue p0.s
-; SVE-NEXT:    sxtb z0.s, p0/m, z0.s
-; SVE-NEXT:    sxtb z1.s, p0/m, z1.s
-; SVE-NEXT:    add z0.s, z0.s, z1.s
-; SVE-NEXT:    asr z0.s, z0.s, #1
-; SVE-NEXT:    ret
-;
-; SVE2-LABEL: hadds_v4i8:
-; SVE2:       // %bb.0: // %entry
-; SVE2-NEXT:    ptrue p0.s
-; SVE2-NEXT:    sxtb z0.s, p0/m, z0.s
-; SVE2-NEXT:    sxtb z1.s, p0/m, z1.s
-; SVE2-NEXT:    shadd z0.s, p0/m, z0.s, z1.s
-; SVE2-NEXT:    ret
+; CHECK-LABEL: hadds_v4i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    sxtb z0.s, p0/m, z0.s
+; CHECK-NEXT:    sxtb z1.s, p0/m, z1.s
+; CHECK-NEXT:    add z0.s, z0.s, z1.s
+; CHECK-NEXT:    asr z0.s, z0.s, #1
+; CHECK-NEXT:    ret
 entry:
   %s0s = sext <vscale x 4 x i8> %s0 to <vscale x 4 x i16>
   %s1s = sext <vscale x 4 x i8> %s1 to <vscale x 4 x i16>
   %m = add nsw <vscale x 4 x i16> %s0s, %s1s
   %s = ashr <vscale x 4 x i16> %m, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 1, i32 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
   %s2 = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>
   ret <vscale x 4 x i8> %s2
 }
entry:
   %s1s = sext <vscale x 4 x i8> %s1 to <vscale x 4 x i16>
   %m = add nsw <vscale x 4 x i16> %s0s, %s1s
   %s = lshr <vscale x 4 x i16> %m, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 1, i32 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
   %s2 = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>
   ret <vscale x 4 x i8> %s2
 }
 
 define <vscale x 4 x i8> @haddu_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {
-; SVE-LABEL: haddu_v4i8:
-; SVE:       // %bb.0: // %entry
-; SVE-NEXT:    and z0.s, z0.s, #0xff
-; SVE-NEXT:    and z1.s, z1.s, #0xff
-; SVE-NEXT:    add z0.s, z0.s, z1.s
-; SVE-NEXT:    lsr z0.s, z0.s, #1
-; SVE-NEXT:    ret
-;
-; SVE2-LABEL: haddu_v4i8:
-; SVE2:       // %bb.0: // %entry
-; SVE2-NEXT:    ptrue p0.s
-; SVE2-NEXT:    and z0.s, z0.s, #0xff
-; SVE2-NEXT:    and z1.s, z1.s, #0xff
-; SVE2-NEXT:    uhadd z0.s, p0/m, z0.s, z1.s
-; SVE2-NEXT:    ret
+; CHECK-LABEL: haddu_v4i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and z0.s, z0.s, #0xff
+; CHECK-NEXT:    and z1.s, z1.s, #0xff
+; CHECK-NEXT:    add z0.s, z0.s, z1.s
+; CHECK-NEXT:    lsr z0.s, z0.s, #1
+; CHECK-NEXT:    ret
 entry:
   %s0s = zext <vscale x 4 x i8> %s0 to <vscale x 4 x i16>
   %s1s = zext <vscale x 4 x i8> %s1 to <vscale x 4 x i16>
   %m = add nuw nsw <vscale x 4 x i16> %s0s, %s1s
   %s = lshr <vscale x 4 x i16> %m, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 1, i32 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
   %s2 = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>
   ret <vscale x 4 x i8> %s2
 }
entry:
   %add = add <vscale x 2 x i32> %s0s, shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
   %add2 = add <vscale x 2 x i32> %add, %s1s
   %s = lshr <vscale x 2 x i32> %add2, shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
   %result = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>
   ret <vscale x 2 x i16> %result
 }
 
 define <vscale x 2 x i16> @rhaddu_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {
-; SVE-LABEL: rhaddu_v2i16:
-; SVE:       // %bb.0: // %entry
-; SVE-NEXT:    mov z2.d, #-1 // =0xffffffffffffffff
-; SVE-NEXT:    and z0.d, z0.d, #0xffff
-; SVE-NEXT:    and z1.d, z1.d, #0xffff
-; SVE-NEXT:    eor z0.d, z0.d, z2.d
-; SVE-NEXT:    sub z0.d, z1.d, z0.d
-; SVE-NEXT:    lsr z0.d, z0.d, #1
-; SVE-NEXT:    ret
-;
-; SVE2-LABEL: rhaddu_v2i16:
-; SVE2:       // %bb.0: // %entry
-; SVE2-NEXT:    ptrue p0.d
-; SVE2-NEXT:    and z0.d, z0.d, #0xffff
-; SVE2-NEXT:    and z1.d, z1.d, #0xffff
-; SVE2-NEXT:    urhadd z0.d, p0/m, z0.d, z1.d
-; SVE2-NEXT:    ret
+; CHECK-LABEL: rhaddu_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z2.d, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    and z0.d, z0.d, #0xffff
+; CHECK-NEXT:    and z1.d, z1.d, #0xffff
+; CHECK-NEXT:    eor z0.d, z0.d, z2.d
+; CHECK-NEXT:    sub z0.d, z1.d, z0.d
+; CHECK-NEXT:    lsr z0.d, z0.d, #1
+; CHECK-NEXT:    ret
 entry:
   %s0s = zext <vscale x 2 x i16> %s0 to <vscale x 2 x i32>
   %s1s = zext <vscale x 2 x i16> %s1 to <vscale x 2 x i32>
   %add = add nuw nsw <vscale x 2 x i32> %s0s, shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
   %add2 = add nuw nsw <vscale x 2 x i32> %add, %s1s
   %s = lshr <vscale x 2 x i32> %add2, shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer)
   %result = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>
   ret <vscale x 2 x i16> %result
entry:
   %add = add <vscale x 4 x i16> %s0s, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 1, i32 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
   %add2 = add <vscale x 4 x i16> %add, %s1s
   %s = lshr <vscale x 4 x i16> %add2, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 1, i32 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
   %result = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>
   ret <vscale x 4 x i8> %result
 }
 
 define <vscale x 4 x i8> @rhaddu_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {
-; SVE-LABEL: rhaddu_v4i8:
-; SVE:       // %bb.0: // %entry
-; SVE-NEXT:    mov z2.s, #-1 // =0xffffffffffffffff
-; SVE-NEXT:    and z0.s, z0.s, #0xff
-; SVE-NEXT:    and z1.s, z1.s, #0xff
-; SVE-NEXT:    eor z0.d, z0.d, z2.d
-; SVE-NEXT:    sub z0.s, z1.s, z0.s
-; SVE-NEXT:    lsr z0.s, z0.s, #1
-; SVE-NEXT:    ret
-;
-; SVE2-LABEL: rhaddu_v4i8:
-; SVE2:       // %bb.0: // %entry
-; SVE2-NEXT:    ptrue p0.s
-; SVE2-NEXT:    and z0.s, z0.s, #0xff
-; SVE2-NEXT:    and z1.s, z1.s, #0xff
-; SVE2-NEXT:    urhadd z0.s, p0/m, z0.s, z1.s
-; SVE2-NEXT:    ret
+; CHECK-LABEL: rhaddu_v4i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z2.s, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    and z0.s, z0.s, #0xff
+; CHECK-NEXT:    and z1.s, z1.s, #0xff
+; CHECK-NEXT:    eor z0.d, z0.d, z2.d
+; CHECK-NEXT:    sub z0.s, z1.s, z0.s
+; CHECK-NEXT:    lsr z0.s, z0.s, #1
+; CHECK-NEXT:    ret
 entry:
   %s0s = zext <vscale x 4 x i8> %s0 to <vscale x 4 x i16>
   %s1s = zext <vscale x 4 x i8> %s1 to <vscale x 4 x i16>
   %add = add nuw nsw <vscale x 4 x i16> %s0s, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 1, i32 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
   %add2 = add nuw nsw <vscale x 4 x i16> %add, %s1s
   %s = lshr <vscale x 4 x i16> %add2, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 1, i32 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
   %result = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>
   ret <vscale x 4 x i8> %result

llvm/test/CodeGen/Thumb2/mve-vhadd.ll

 }
 
 define arm_aapcs_vfpcc <4 x i8> @vhaddu_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
 ; CHECK-LABEL: vhaddu_v4i8:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmov.i32 q2, #0xff
 ; CHECK-NEXT:    vand q1, q1, q2
 ; CHECK-NEXT:    vand q0, q0, q2
-; CHECK-NEXT:    vhadd.u32 q0, q0, q1
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vshr.u32 q0, q0, #1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = zext <4 x i8> %s0 to <4 x i16>
   %s1s = zext <4 x i8> %s1 to <4 x i16>
   %m = add nuw nsw <4 x i16> %s0s, %s1s
   %s = lshr <4 x i16> %m, <i16 1, i16 1, i16 1, i16 1>
   %s2 = trunc <4 x i16> %s to <4 x i8>
   ret <4 x i8> %s2
entry:
   %result = trunc <4 x i16> %s to <4 x i8>
   ret <4 x i8> %result
 }
 
 define arm_aapcs_vfpcc <4 x i8> @vrhaddu_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
 ; CHECK-LABEL: vrhaddu_v4i8:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmov.i32 q2, #0xff
+; CHECK-NEXT:    movs r0, #1
 ; CHECK-NEXT:    vand q1, q1, q2
 ; CHECK-NEXT:    vand q0, q0, q2
-; CHECK-NEXT:    vrhadd.u32 q0, q0, q1
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vshr.u32 q0, q0, #1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = zext <4 x i8> %s0 to <4 x i16>
   %s1s = zext <4 x i8> %s1 to <4 x i16>
   %add = add nuw nsw <4 x i16> %s0s, <i16 1, i16 1, i16 1, i16 1>
   %add2 = add nuw nsw <4 x i16> %add, %s1s
   %s = lshr <4 x i16> %add2, <i16 1, i16 1, i16 1, i16 1>
   %result = trunc <4 x i16> %s to <4 x i8>