diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -5842,31 +5842,55 @@ OutputZeroVal = FalseVal; if (OutputZeroVal) { - if (match(CmpLHS, m_AnyZeroFP())) - CmpLHS = OutputZeroVal; - if (match(CmpRHS, m_AnyZeroFP())) - CmpRHS = OutputZeroVal; + // Tries to adjust 'Pred' so that we can flip the sign of one of its zero + // operands. SideAndSignBit must be true iff the LHS is positive or iff + // the RHS operand is negative. Applies the following equivalences: + // >= +0 <=> > -0 and -0 >= <=> +0 > + // <= -0 <=> < +0 and +0 <= <=> -0 < + auto MaybeAdjustPredicate = [&](bool SideAndSignBit) { + switch (Pred) { + case CmpInst::FCMP_OGT: + return SideAndSignBit ? (Pred = CmpInst::FCMP_OGE, true) : false; + case CmpInst::FCMP_UGT: + return SideAndSignBit ? (Pred = CmpInst::FCMP_UGE, true) : false; + case CmpInst::FCMP_OLE: + return SideAndSignBit ? (Pred = CmpInst::FCMP_OLT, true) : false; + case CmpInst::FCMP_ULE: + return SideAndSignBit ? (Pred = CmpInst::FCMP_ULT, true) : false; + case CmpInst::FCMP_OGE: + return SideAndSignBit ? false : (Pred = CmpInst::FCMP_OGT, true); + case CmpInst::FCMP_UGE: + return SideAndSignBit ? false : (Pred = CmpInst::FCMP_UGT, true); + case CmpInst::FCMP_OLT: + return SideAndSignBit ? false : (Pred = CmpInst::FCMP_OLE, true); + case CmpInst::FCMP_ULT: + return SideAndSignBit ? false : (Pred = CmpInst::FCMP_ULE, true); + default: + return false; + } + }; + + bool ZeroSignBit = match(OutputZeroVal, m_NegZeroFP()); + if (match(CmpLHS, m_AnyZeroFP())) { + bool SignLHS = match(CmpLHS, m_NegZeroFP()); + if (FMF.noSignedZeros() || ZeroSignBit == SignLHS || + MaybeAdjustPredicate(SignLHS == false)) { + CmpLHS = OutputZeroVal; + } + } + if (match(CmpRHS, m_AnyZeroFP())) { + bool SignRHS = match(CmpLHS, m_NegZeroFP()); + if (FMF.noSignedZeros() || ZeroSignBit == SignRHS || + MaybeAdjustPredicate(SignRHS == true)) { + CmpRHS = OutputZeroVal; + } + } } } LHS = CmpLHS; RHS = CmpRHS; - // Signed zero may return inconsistent results between implementations. - // (0.0 <= -0.0) ? 0.0 : -0.0 // Returns 0.0 - // minNum(0.0, -0.0) // May return -0.0 or 0.0 (IEEE 754-2008 5.3.1) - // Therefore, we behave conservatively and only proceed if at least one of the - // operands is known to not be zero or if we don't care about signed zero. - switch (Pred) { - default: break; - // FIXME: Include OGT/OLT/UGT/ULT. - case CmpInst::FCMP_OGE: case CmpInst::FCMP_OLE: - case CmpInst::FCMP_UGE: case CmpInst::FCMP_ULE: - if (!FMF.noSignedZeros() && !isKnownNonZero(CmpLHS) && - !isKnownNonZero(CmpRHS)) - return {SPF_UNKNOWN, SPNB_NA, false}; - } - SelectPatternNaNBehavior NaNBehavior = SPNB_NA; bool Ordered = false; @@ -5919,6 +5943,33 @@ Ordered = !Ordered; } + switch (Pred) { + default: + break; + case CmpInst::FCMP_OGT: + case CmpInst::FCMP_OGE: + case CmpInst::FCMP_OLT: + case CmpInst::FCMP_OLE: + case CmpInst::FCMP_UGT: + case CmpInst::FCMP_UGE: + case CmpInst::FCMP_ULT: + case CmpInst::FCMP_ULE: + if (!FMF.noSignedZeros() && !isKnownNonZero(CmpLHS) && + !isKnownNonZero(CmpRHS)) { + // According to (IEEE 754-2008 5.3.1), minNum(0.0, -0.0) and similar may + // return either -0.0 or 0.0, so fcmp/select pair has stricter semantics + // than minNum. Bail out unless at least one operand is known to be + // non-zero or we don't care about signed zeros. + if (NaNBehavior == SPNB_RETURNS_OTHER) + return {SPF_UNKNOWN, SPNB_NA, false}; + // Prevent combining into minNum/maxNum because of the stricter semantics + // described above, but allow combining into minimum/maximum, which does + // guarantee that -0.0 is less than 0.0. + if (NaNBehavior == SPNB_RETURNS_ANY) + NaNBehavior = SPNB_RETURNS_NAN; + } + } + // ([if]cmp X, Y) ? X : Y if (TrueVal == CmpLHS && FalseVal == CmpRHS) { switch (Pred) { @@ -5994,12 +6045,7 @@ if (CmpInst::isIntPredicate(Pred)) return matchMinMax(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS, Depth); - // According to (IEEE 754-2008 5.3.1), minNum(0.0, -0.0) and similar - // may return either -0.0 or 0.0, so fcmp/select pair has stricter - // semantics than minNum. Be conservative in such case. - if (NaNBehavior != SPNB_RETURNS_ANY || - (!FMF.noSignedZeros() && !isKnownNonZero(CmpLHS) && - !isKnownNonZero(CmpRHS))) + if (NaNBehavior != SPNB_RETURNS_ANY) return {SPF_UNKNOWN, SPNB_NA, false}; return matchFastFloatClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS); diff --git a/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll b/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll --- a/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll +++ b/llvm/test/CodeGen/AArch64/arm64-fmax-safe.ll @@ -20,11 +20,11 @@ ; CHECK: fmin s } -; Same as previous, but with ordered comparison; +; Same as previous, but with ordered comparison and no-signed-zero; ; must become fminnm, not fmin. define double @test_cross_fail_nan(float %in) { ; CHECK-LABEL: test_cross_fail_nan: - %cmp = fcmp olt float %in, 0.000000e+00 + %cmp = fcmp nsz olt float %in, 0.000000e+00 %val = select i1 %cmp, float %in, float 0.000000e+00 %longer = fpext float %val to double ret double %longer diff --git a/llvm/test/CodeGen/ARM/fp16-vminmaxnm-safe.ll b/llvm/test/CodeGen/ARM/fp16-vminmaxnm-safe.ll --- a/llvm/test/CodeGen/ARM/fp16-vminmaxnm-safe.ll +++ b/llvm/test/CodeGen/ARM/fp16-vminmaxnm-safe.ll @@ -493,7 +493,7 @@ ; CHECK-NEXT: .LCPI23_0: ; CHECK-NEXT: .short 0x8000 @ half -0 entry: - %cmp1 = fcmp olt half %a, -0. + %cmp1 = fcmp nsz olt half %a, -0. %cond1 = select i1 %cmp1, half %a, half -0. %cmp2 = fcmp ugt half %cond1, -0. %cond2 = select i1 %cmp2, half %cond1, half -0. diff --git a/llvm/test/CodeGen/ARM/neon_minmax.ll b/llvm/test/CodeGen/ARM/neon_minmax.ll --- a/llvm/test/CodeGen/ARM/neon_minmax.ll +++ b/llvm/test/CodeGen/ARM/neon_minmax.ll @@ -11,7 +11,7 @@ define float @fmin_ole_zero(float %x) nounwind { ;CHECK-LABEL: fmin_ole_zero: -;CHECK-NOT: vmin.f32 +;CHECK: vmin.f32 %cond = fcmp ole float 0.0, %x %min1 = select i1 %cond, float 0.0, float %x ret float %min1 @@ -43,7 +43,7 @@ define float @fmax_uge_zero(float %x) nounwind { ;CHECK-LABEL: fmax_uge_zero: -;CHECK-NOT: vmax.f32 +;CHECK: vmax.f32 %cond = fcmp uge float %x, 0.0 %max1 = select i1 %cond, float %x, float 0.0 ret float %max1 diff --git a/llvm/test/CodeGen/ARM/vminmaxnm-safe.ll b/llvm/test/CodeGen/ARM/vminmaxnm-safe.ll --- a/llvm/test/CodeGen/ARM/vminmaxnm-safe.ll +++ b/llvm/test/CodeGen/ARM/vminmaxnm-safe.ll @@ -350,9 +350,9 @@ ; CHECK-LABEL: "fp-armv8_vminmaxnm_0": ; CHECK-NOT: vminnm.f32 ; CHECK: vmaxnm.f32 - %cmp1 = fcmp ult float %a, 0. + %cmp1 = fcmp nsz ult float %a, 0. %cond1 = select i1 %cmp1, float %a, float 0. - %cmp2 = fcmp ogt float %cond1, 0. + %cmp2 = fcmp nsz ogt float %cond1, 0. %cond2 = select i1 %cmp2, float %cond1, float 0. ret float %cond2 } @@ -361,7 +361,7 @@ ; CHECK-LABEL: "fp-armv8_vminmaxnm_neg0": ; CHECK: vminnm.f32 ; CHECK-NOT: vmaxnm.f32 - %cmp1 = fcmp olt float %a, -0. + %cmp1 = fcmp nsz olt float %a, -0. %cond1 = select i1 %cmp1, float %a, float -0. %cmp2 = fcmp ugt float %cond1, -0. %cond2 = select i1 %cmp2, float %cond1, float -0. diff --git a/llvm/test/Transforms/InstCombine/minmax-fp.ll b/llvm/test/Transforms/InstCombine/minmax-fp.ll --- a/llvm/test/Transforms/InstCombine/minmax-fp.ll +++ b/llvm/test/Transforms/InstCombine/minmax-fp.ll @@ -78,7 +78,7 @@ define double @t7(float %a) { ; CHECK-LABEL: @t7( -; CHECK-NEXT: [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 0.000000e+00 +; CHECK-NEXT: [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], -0.000000e+00 ; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[DOTINV]], float -0.000000e+00, float [[A]] ; CHECK-NEXT: [[TMP2:%.*]] = fpext float [[TMP1]] to double ; CHECK-NEXT: ret double [[TMP2]] @@ -108,13 +108,13 @@ define float @fmax_fmax_zero_mismatch(float %x) { ; CHECK-LABEL: @fmax_fmax_zero_mismatch( -; CHECK-NEXT: [[TMP1:%.*]] = fcmp ogt float [[X:%.*]], -0.000000e+00 +; CHECK-NEXT: [[TMP1:%.*]] = fcmp nsz ogt float [[X:%.*]], -0.000000e+00 ; CHECK-NEXT: [[MAX11:%.*]] = select i1 [[TMP1]], float [[X]], float -0.000000e+00 ; CHECK-NEXT: ret float [[MAX11]] ; - %cmp1 = fcmp ogt float %x, 0.0 + %cmp1 = fcmp nsz ogt float %x, 0.0 %max1 = select i1 %cmp1, float %x, float -0.0 - %cmp2 = fcmp ogt float 0.0, %max1 + %cmp2 = fcmp nsz ogt float 0.0, %max1 %max2 = select i1 %cmp2, float -0.0, float %max1 ret float %max2 } diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp --- a/llvm/unittests/Analysis/ValueTrackingTest.cpp +++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp @@ -224,7 +224,7 @@ " ret float %A\n" "}\n"); // The sign of zero doesn't matter in fcmp. - expectPattern({SPF_FMINNUM, SPNB_RETURNS_NAN, true}); + expectPattern({SPF_UNKNOWN, SPNB_NA, false}); } TEST_F(MatchSelectPatternTest, FMinMismatchConstantZero4) { @@ -235,7 +235,7 @@ " ret float %A\n" "}\n"); // The sign of zero doesn't matter in fcmp. - expectPattern({SPF_FMINNUM, SPNB_RETURNS_NAN, false}); + expectPattern({SPF_UNKNOWN, SPNB_NA, false}); } TEST_F(MatchSelectPatternTest, FMinMismatchConstantZero5) { @@ -246,7 +246,7 @@ " ret float %A\n" "}\n"); // The sign of zero doesn't matter in fcmp. - expectPattern({SPF_FMINNUM, SPNB_RETURNS_OTHER, false}); + expectPattern({SPF_UNKNOWN, SPNB_NA, false}); } TEST_F(MatchSelectPatternTest, FMinMismatchConstantZero6) { @@ -257,7 +257,7 @@ " ret float %A\n" "}\n"); // The sign of zero doesn't matter in fcmp. - expectPattern({SPF_FMINNUM, SPNB_RETURNS_OTHER, true}); + expectPattern({SPF_UNKNOWN, SPNB_NA, false}); } TEST_F(MatchSelectPatternTest, FMinMismatchConstantZero7) { @@ -268,7 +268,7 @@ " ret float %A\n" "}\n"); // The sign of zero doesn't matter in fcmp. - expectPattern({SPF_FMINNUM, SPNB_RETURNS_OTHER, false}); + expectPattern({SPF_UNKNOWN, SPNB_NA, false}); } TEST_F(MatchSelectPatternTest, FMinMismatchConstantZero8) { @@ -279,7 +279,7 @@ " ret float %A\n" "}\n"); // The sign of zero doesn't matter in fcmp. - expectPattern({SPF_FMINNUM, SPNB_RETURNS_OTHER, true}); + expectPattern({SPF_UNKNOWN, SPNB_NA, false}); } TEST_F(MatchSelectPatternTest, FMaxMismatchConstantZero1) { @@ -290,7 +290,7 @@ " ret float %A\n" "}\n"); // The sign of zero doesn't matter in fcmp. - expectPattern({SPF_FMAXNUM, SPNB_RETURNS_NAN, true}); + expectPattern({SPF_UNKNOWN, SPNB_NA, false}); } TEST_F(MatchSelectPatternTest, FMaxMismatchConstantZero2) { @@ -334,7 +334,7 @@ " ret float %A\n" "}\n"); // The sign of zero doesn't matter in fcmp. - expectPattern({SPF_FMAXNUM, SPNB_RETURNS_OTHER, false}); + expectPattern({SPF_UNKNOWN, SPNB_NA, false}); } TEST_F(MatchSelectPatternTest, FMaxMismatchConstantZero6) { @@ -345,7 +345,7 @@ " ret float %A\n" "}\n"); // The sign of zero doesn't matter in fcmp. - expectPattern({SPF_FMAXNUM, SPNB_RETURNS_OTHER, true}); + expectPattern({SPF_UNKNOWN, SPNB_NA, false}); } TEST_F(MatchSelectPatternTest, FMaxMismatchConstantZero7) { @@ -356,7 +356,7 @@ " ret float %A\n" "}\n"); // The sign of zero doesn't matter in fcmp. - expectPattern({SPF_FMAXNUM, SPNB_RETURNS_OTHER, false}); + expectPattern({SPF_UNKNOWN, SPNB_NA, false}); } TEST_F(MatchSelectPatternTest, FMaxMismatchConstantZero8) { @@ -367,7 +367,7 @@ " ret float %A\n" "}\n"); // The sign of zero doesn't matter in fcmp. - expectPattern({SPF_FMAXNUM, SPNB_RETURNS_OTHER, true}); + expectPattern({SPF_UNKNOWN, SPNB_NA, false}); } TEST_F(MatchSelectPatternTest, FMinMismatchConstantZeroVecUndef) {