diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3111,6 +3111,8 @@ void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) { SDNodeFlags Flags; + if (auto *FPOp = dyn_cast(&I)) + Flags.copyFMF(*FPOp); SDValue Op = getValue(I.getOperand(0)); SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(), diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -19117,7 +19117,7 @@ if (isMergePassthruOpcode(NewOp)) Operands.push_back(DAG.getUNDEF(VT)); - return DAG.getNode(NewOp, DL, VT, Operands); + return DAG.getNode(NewOp, DL, VT, Operands, Op->getFlags()); } // If a fixed length vector operation has no side effects when applied to diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -275,6 +275,11 @@ return N->hasOneUse(); }]>; +def AArch64fneg_mt_nsz : PatFrag<(ops node:$pred, node:$op, node:$pt), + (AArch64fneg_mt node:$pred, node:$op, node:$pt), [{ + return N->getFlags().hasNoSignedZeros(); +}]>; + def SDT_AArch64Arith_Unpred : SDTypeProfile<1, 2, [ SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisSameAs<0,1>, SDTCisSameAs<1,2> @@ -536,7 +541,8 @@ (!cast("FNMLA_ZPZZZ_UNDEF_"#Suffix) $P, ZPR:$Za, ZPR:$Zn, ZPR:$Zm)>; // Zd = -(Za + Zn * Zm) - def : Pat<(AArch64fneg_mt PredTy:$P, (AArch64fma_p PredTy:$P, Ty:$Zn, Ty:$Zm, Ty:$Za), (Ty (undef))), + // (with nsz neg.) + def : Pat<(AArch64fneg_mt_nsz PredTy:$P, (AArch64fma_p PredTy:$P, Ty:$Zn, Ty:$Zm, Ty:$Za), (Ty (undef))), (!cast("FNMLA_ZPZZZ_UNDEF_"#Suffix) $P, ZPR:$Za, ZPR:$Zn, ZPR:$Zm)>; // Zda = Zda + Zn * Zm diff --git a/llvm/test/CodeGen/AArch64/sve-fp-combine.ll b/llvm/test/CodeGen/AArch64/sve-fp-combine.ll --- a/llvm/test/CodeGen/AArch64/sve-fp-combine.ll +++ b/llvm/test/CodeGen/AArch64/sve-fp-combine.ll @@ -549,7 +549,7 @@ ; CHECK-NEXT: ret %mul = fmul contract %m1, %m2 %add = fadd contract %mul, %acc - %res = fneg contract %add + %res = fneg contract nsz %add ret %res } @@ -561,7 +561,7 @@ ; CHECK-NEXT: ret %mul = fmul contract %m1, %m2 %add = fadd contract %mul, %acc - %res = fneg contract %add + %res = fneg contract nsz %add ret %res } @@ -573,7 +573,7 @@ ; CHECK-NEXT: ret %mul = fmul contract %m1, %m2 %add = fadd contract %mul, %acc - %res = fneg contract %add + %res = fneg contract nsz %add ret %res } @@ -585,7 +585,7 @@ ; CHECK-NEXT: ret %mul = fmul contract %m1, %m2 %add = fadd contract %mul, %acc - %res = fneg contract %add + %res = fneg contract nsz %add ret %res } @@ -597,7 +597,7 @@ ; CHECK-NEXT: ret %mul = fmul contract %m1, %m2 %add = fadd contract %mul, %acc - %res = fneg contract %add + %res = fneg contract nsz %add ret %res } @@ -606,6 +606,84 @@ ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: fnmla z0.d, p0/m, z1.d, z2.d +; CHECK-NEXT: ret + %mul = fmul contract %m1, %m2 + %add = fadd contract %mul, %acc + %res = fneg contract nsz %add + ret %res +} + +define @signed_zeros_negtest_fnmla_h_reversed( %acc, %m1, %m2) { +; CHECK-LABEL: signed_zeros_negtest_fnmla_h_reversed: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: fmla z0.h, p0/m, z1.h, z2.h +; CHECK-NEXT: fneg z0.h, p0/m, z0.h +; CHECK-NEXT: ret + %mul = fmul contract %m1, %m2 + %add = fadd contract %mul, %acc + %res = fneg contract %add + ret %res +} + +define @signed_zeros_negtest_fnmla_hx4_reversed( %acc, %m1, %m2) { +; CHECK-LABEL: signed_zeros_negtest_fnmla_hx4_reversed: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: fmla z0.h, p0/m, z1.h, z2.h +; CHECK-NEXT: fneg z0.h, p0/m, z0.h +; CHECK-NEXT: ret + %mul = fmul contract %m1, %m2 + %add = fadd contract %mul, %acc + %res = fneg contract %add + ret %res +} + +define @signed_zeros_negtest_fnmla_hx2_reversed( %acc, %m1, %m2) { +; CHECK-LABEL: signed_zeros_negtest_fnmla_hx2_reversed: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: fmla z0.h, p0/m, z1.h, z2.h +; CHECK-NEXT: fneg z0.h, p0/m, z0.h +; CHECK-NEXT: ret + %mul = fmul contract %m1, %m2 + %add = fadd contract %mul, %acc + %res = fneg contract %add + ret %res +} + +define @signed_zeros_negtest_fnmla_s_reversed( %acc, %m1, %m2) { +; CHECK-LABEL: signed_zeros_negtest_fnmla_s_reversed: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: fmla z0.s, p0/m, z1.s, z2.s +; CHECK-NEXT: fneg z0.s, p0/m, z0.s +; CHECK-NEXT: ret + %mul = fmul contract %m1, %m2 + %add = fadd contract %mul, %acc + %res = fneg contract %add + ret %res +} + +define @signed_zeros_negtest_fnmla_sx2_reversed( %acc, %m1, %m2) { +; CHECK-LABEL: signed_zeros_negtest_fnmla_sx2_reversed: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: fmla z0.s, p0/m, z1.s, z2.s +; CHECK-NEXT: fneg z0.s, p0/m, z0.s +; CHECK-NEXT: ret + %mul = fmul contract %m1, %m2 + %add = fadd contract %mul, %acc + %res = fneg contract %add + ret %res +} + +define @signed_zeros_negtest_fnmla_d_reversed( %acc, %m1, %m2) { +; CHECK-LABEL: signed_zeros_negtest_fnmla_d_reversed: +; CHECK: // %bb.0: +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: fmla z0.d, p0/m, z1.d, z2.d +; CHECK-NEXT: fneg z0.d, p0/m, z0.d ; CHECK-NEXT: ret %mul = fmul contract %m1, %m2 %add = fadd contract %mul, %acc