Index: llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp =================================================================== --- llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -2929,10 +2929,8 @@ // Canonicalize select with fcmp to fabs(). -0.0 makes this tricky. We need // fast-math-flags (nsz) or fsub with +0.0 (not fneg) for this to work. // (X <= +/-0.0) ? (0.0 - X) : X --> fabs(X) - Instruction *FSub; if (match(CondVal, m_FCmp(Pred, m_Specific(FalseVal), m_AnyZeroFP())) && match(TrueVal, m_FSub(m_PosZeroFP(), m_Specific(FalseVal))) && - match(TrueVal, m_Instruction(FSub)) && (Pred == FCmpInst::FCMP_OLE || Pred == FCmpInst::FCMP_ULE)) { Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, FalseVal, &SI); return replaceInstUsesWith(SI, Fabs); @@ -2940,35 +2938,62 @@ // (X > +/-0.0) ? X : (0.0 - X) --> fabs(X) if (match(CondVal, m_FCmp(Pred, m_Specific(TrueVal), m_AnyZeroFP())) && match(FalseVal, m_FSub(m_PosZeroFP(), m_Specific(TrueVal))) && - match(FalseVal, m_Instruction(FSub)) && (Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_UGT)) { Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, TrueVal, &SI); return replaceInstUsesWith(SI, Fabs); } - // With nnan and nsz: + // (X < +/-0.0) ? X : (0.0 - X) --> (0.0 - fabs(X)) + // With nsz: + // (X < +/-0.0) ? X : -X --> -fabs(X) + if (match(CondVal, m_FCmp(Pred, m_Specific(TrueVal), m_AnyZeroFP())) && + (match(FalseVal, m_FSub(m_PosZeroFP(), m_Specific(TrueVal))) || + (match(FalseVal, m_FNeg(m_Specific(TrueVal))) && + cast(FalseVal)->hasNoSignedZeros())) && + (Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_ULT)) { + Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, TrueVal, &SI); + return BinaryOperator::CreateFSubFMF(ConstantFP::get(Fabs->getType(), 0.0), + Fabs, cast(FalseVal)); + } + // With nsz: // (X < +/-0.0) ? -X : X --> fabs(X) // (X <= +/-0.0) ? -X : X --> fabs(X) - Instruction *FNeg; if (match(CondVal, m_FCmp(Pred, m_Specific(FalseVal), m_AnyZeroFP())) && - match(TrueVal, m_FNeg(m_Specific(FalseVal))) && - match(TrueVal, m_Instruction(FNeg)) && SI.hasNoSignedZeros() && + match(TrueVal, m_FNeg(m_Specific(FalseVal))) && SI.hasNoSignedZeros() && (Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_OLE || Pred == FCmpInst::FCMP_ULT || Pred == FCmpInst::FCMP_ULE)) { Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, FalseVal, &SI); return replaceInstUsesWith(SI, Fabs); } - // With nnan and nsz: + // With nsz: // (X > +/-0.0) ? X : -X --> fabs(X) // (X >= +/-0.0) ? X : -X --> fabs(X) if (match(CondVal, m_FCmp(Pred, m_Specific(TrueVal), m_AnyZeroFP())) && - match(FalseVal, m_FNeg(m_Specific(TrueVal))) && - match(FalseVal, m_Instruction(FNeg)) && SI.hasNoSignedZeros() && + match(FalseVal, m_FNeg(m_Specific(TrueVal))) && SI.hasNoSignedZeros() && (Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_OGE || Pred == FCmpInst::FCMP_UGT || Pred == FCmpInst::FCMP_UGE)) { Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, TrueVal, &SI); return replaceInstUsesWith(SI, Fabs); } - + // With nsz: + // (X < +/-0.0) ? X : -X --> -fabs(X) + // (X <= +/-0.0) ? X : -X --> -fabs(X) + if (match(CondVal, m_FCmp(Pred, m_Specific(TrueVal), m_AnyZeroFP())) && + match(FalseVal, m_FNeg(m_Specific(TrueVal))) && SI.hasNoSignedZeros() && + (Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_OLE || + Pred == FCmpInst::FCMP_ULT || Pred == FCmpInst::FCMP_ULE)) { + Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, TrueVal, &SI); + return UnaryOperator::CreateFNegFMF(Fabs, cast(FalseVal)); + } + // With nsz: + // (X > +/-0.0) ? -X : X --> -fabs(X) + // (X >= +/-0.0) ? -X : X --> -fabs(X) + if (match(CondVal, m_FCmp(Pred, m_Specific(FalseVal), m_AnyZeroFP())) && + match(TrueVal, m_FNeg(m_Specific(FalseVal))) && SI.hasNoSignedZeros() && + (Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_OGE || + Pred == FCmpInst::FCMP_UGT || Pred == FCmpInst::FCMP_UGE)) { + Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, FalseVal, &SI); + return UnaryOperator::CreateFNegFMF(Fabs, cast(TrueVal)); + } // See if we are selecting two values based on a comparison of the two values. if (ICmpInst *ICI = dyn_cast(CondVal)) if (Instruction *Result = foldSelectInstWithICmp(SI, ICI)) Index: llvm/test/Transforms/InstCombine/fabs.ll =================================================================== --- llvm/test/Transforms/InstCombine/fabs.ll +++ llvm/test/Transforms/InstCombine/fabs.ll @@ -975,3 +975,225 @@ %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %s) ret <2 x float> %fabs } + +; (X < +/-0.0) ? X : -X --> -fabs(X) +; (X <= +/-0.0) ? X : -X --> -fabs(X) +; One negative test with no fmf +define double @select_noFMF_nfabs_lt(double %x) { +; CHECK-LABEL: @select_noFMF_nfabs_lt( +; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[X:%.*]], 0.000000e+00 +; CHECK-NEXT: [[NEGX:%.*]] = fneg double [[X]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], double [[X]], double [[NEGX]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp = fcmp olt double %x, 0.000000e+00 + %negX = fneg double %x + %sel = select i1 %cmp, double %x, double %negX + ret double %sel +} + +; One test where the neg has fmfs. +define double @select_nsz_nfabs_lt_fmfProp(double %x) { +; CHECK-LABEL: @select_nsz_nfabs_lt_fmfProp( +; CHECK-NEXT: [[TMP1:%.*]] = call nsz double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[SEL:%.*]] = fneg fast double [[TMP1]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp = fcmp olt double %x, 0.000000e+00 + %negX = fneg fast double %x + %sel = select nsz i1 %cmp, double %x, double %negX + ret double %sel +} + +; Tests with various predicate types. +define double @select_nsz_nfabs_olt(double %x) { +; CHECK-LABEL: @select_nsz_nfabs_olt( +; CHECK-NEXT: [[TMP1:%.*]] = call nsz double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[SEL:%.*]] = fneg double [[TMP1]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp = fcmp olt double %x, 0.000000e+00 + %negX = fneg double %x + %sel = select nsz i1 %cmp, double %x, double %negX + ret double %sel +} + +define double @select_nsz_nfabs_ult(double %x) { +; CHECK-LABEL: @select_nsz_nfabs_ult( +; CHECK-NEXT: [[TMP1:%.*]] = call nsz double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[SEL:%.*]] = fneg double [[TMP1]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp = fcmp ult double %x, 0.000000e+00 + %negX = fneg double %x + %sel = select nsz i1 %cmp, double %x, double %negX + ret double %sel +} + +define double @select_nsz_nfabs_ole(double %x) { +; CHECK-LABEL: @select_nsz_nfabs_ole( +; CHECK-NEXT: [[TMP1:%.*]] = call nsz double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[SEL:%.*]] = fneg double [[TMP1]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp = fcmp ole double %x, 0.000000e+00 + %negX = fneg double %x + %sel = select nsz i1 %cmp, double %x, double %negX + ret double %sel +} + +define double @select_nsz_nfabs_ule(double %x) { +; CHECK-LABEL: @select_nsz_nfabs_ule( +; CHECK-NEXT: [[TMP1:%.*]] = call nsz double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[SEL:%.*]] = fneg double [[TMP1]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp = fcmp ule double %x, 0.000000e+00 + %negX = fneg double %x + %sel = select nsz i1 %cmp, double %x, double %negX + ret double %sel +} + +; (X > +/-0.0) ? -X : X --> -fabs(X) +; (X >= +/-0.0) ? -X : X --> -fabs(X) +; One negative test with no fmf +define double @select_noFMF_nfabs_gt(double %x) { +; CHECK-LABEL: @select_noFMF_nfabs_gt( +; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt double [[X:%.*]], 0.000000e+00 +; CHECK-NEXT: [[NEGX:%.*]] = fneg double [[X]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], double [[NEGX]], double [[X]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp = fcmp ogt double %x, 0.000000e+00 + %negX = fneg double %x + %sel = select i1 %cmp, double %negX, double %x + ret double %sel +} + +; One test where the neg has fmfs. +define double @select_nsz_nfabs_gt_fmfProp(double %x) { +; CHECK-LABEL: @select_nsz_nfabs_gt_fmfProp( +; CHECK-NEXT: [[TMP1:%.*]] = call nsz double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[SEL:%.*]] = fneg fast double [[TMP1]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp = fcmp ogt double %x, 0.000000e+00 + %negX = fneg fast double %x + %sel = select nsz i1 %cmp, double %negX, double %x + ret double %sel +} + +; Tests with various predicate types. +define double @select_nsz_nfabs_ogt(double %x) { +; CHECK-LABEL: @select_nsz_nfabs_ogt( +; CHECK-NEXT: [[TMP1:%.*]] = call nsz double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[SEL:%.*]] = fneg double [[TMP1]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp = fcmp ogt double %x, 0.000000e+00 + %negX = fneg double %x + %sel = select nsz i1 %cmp, double %negX, double %x + ret double %sel +} + +define double @select_nsz_nfabs_ugt(double %x) { +; CHECK-LABEL: @select_nsz_nfabs_ugt( +; CHECK-NEXT: [[TMP1:%.*]] = call nsz double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[SEL:%.*]] = fneg double [[TMP1]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp = fcmp ugt double %x, 0.000000e+00 + %negX = fneg double %x + %sel = select nsz i1 %cmp, double %negX, double %x + ret double %sel +} + +define double @select_nsz_nfabs_oge(double %x) { +; CHECK-LABEL: @select_nsz_nfabs_oge( +; CHECK-NEXT: [[TMP1:%.*]] = call nsz double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[SEL:%.*]] = fneg double [[TMP1]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp = fcmp oge double %x, 0.000000e+00 + %negX = fneg double %x + %sel = select nsz i1 %cmp, double %negX, double %x + ret double %sel +} + +define double @select_nsz_nfabs_uge(double %x) { +; CHECK-LABEL: @select_nsz_nfabs_uge( +; CHECK-NEXT: [[TMP1:%.*]] = call nsz double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[SEL:%.*]] = fneg double [[TMP1]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp = fcmp uge double %x, 0.000000e+00 + %negX = fneg double %x + %sel = select nsz i1 %cmp, double %negX, double %x + ret double %sel +} + +; (X < +/-0.0) ? X : (0.0 - X) --> (0.0 - fabs(X)) +; One negative test with <=. +define double @select_noFMF_fsubfabs_le(double %x) { +; CHECK-LABEL: @select_noFMF_fsubfabs_le( +; CHECK-NEXT: [[CMP:%.*]] = fcmp ole double [[X:%.*]], 0.000000e+00 +; CHECK-NEXT: [[SUB:%.*]] = fsub double 0.000000e+00, [[X]] +; CHECK-NEXT: [[RETVAL_0:%.*]] = select i1 [[CMP]], double [[X]], double [[SUB]] +; CHECK-NEXT: ret double [[RETVAL_0]] +; + %cmp = fcmp ole double %x, 0.000000e+00 + %sub = fsub double 0.000000e+00, %x + %retval.0 = select i1 %cmp, double %x, double %sub + ret double %retval.0 +} + +define double @select_noFMF_fsubfabs_olt(double %x) { +; CHECK-LABEL: @select_noFMF_fsubfabs_olt( +; CHECK-NEXT: [[TMP1:%.*]] = call double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[RETVAL_0:%.*]] = fsub double 0.000000e+00, [[TMP1]] +; CHECK-NEXT: ret double [[RETVAL_0]] +; + %cmp = fcmp olt double %x, 0.000000e+00 + %sub = fsub double 0.000000e+00, %x + %retval.0 = select i1 %cmp, double %x, double %sub + ret double %retval.0 +} + +define double @select_noFMF_fsubfabs_ult(double %x) { +; CHECK-LABEL: @select_noFMF_fsubfabs_ult( +; CHECK-NEXT: [[TMP1:%.*]] = call double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[RETVAL_0:%.*]] = fsub double 0.000000e+00, [[TMP1]] +; CHECK-NEXT: ret double [[RETVAL_0]] +; + %cmp = fcmp ult double %x, 0.000000e+00 + %sub = fsub double 0.000000e+00, %x + %retval.0 = select i1 %cmp, double %x, double %sub + ret double %retval.0 +} + + +; With nsz: +; (X < +/-0.0) ? X : -X --> -fabs(X) +define double @select_nsz_fnegfabs_olt(double %x) { +; CHECK-LABEL: @select_nsz_fnegfabs_olt( +; CHECK-NEXT: [[TMP1:%.*]] = call double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[RETVAL_0:%.*]] = fneg nsz double [[TMP1]] +; CHECK-NEXT: ret double [[RETVAL_0]] +; + %cmp = fcmp olt double %x, 0.000000e+00 + %negX = fneg nsz double %x + %retval.0 = select i1 %cmp, double %x, double %negX + ret double %retval.0 +} + +define double @select_nsz_fnegfabs_ult(double %x) { +; CHECK-LABEL: @select_nsz_fnegfabs_ult( +; CHECK-NEXT: [[TMP1:%.*]] = call double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[RETVAL_0:%.*]] = fneg nsz double [[TMP1]] +; CHECK-NEXT: ret double [[RETVAL_0]] +; + %cmp = fcmp ult double %x, 0.000000e+00 + %negX = fneg nsz double %x + %retval.0 = select i1 %cmp, double %x, double %negX + ret double %retval.0 +}