diff --git a/llvm/include/llvm/Support/DivisionByConstantInfo.h b/llvm/include/llvm/Support/DivisionByConstantInfo.h --- a/llvm/include/llvm/Support/DivisionByConstantInfo.h +++ b/llvm/include/llvm/Support/DivisionByConstantInfo.h @@ -26,11 +26,13 @@ /// Magic data for optimising unsigned division by a constant. struct UnsignedDivisionByConstantInfo { - static UnsignedDivisionByConstantInfo get(const APInt &D, - unsigned LeadingZeros = 0); + static UnsignedDivisionByConstantInfo + get(const APInt &D, unsigned LeadingZeros = 0, + bool AllowEvenDivisorOptimization = true); APInt Magic; ///< magic number bool IsAdd; ///< add indicator unsigned ShiftAmount; ///< shift amount + unsigned PreShift; ///< pre-shift amount }; } // namespace llvm diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -4952,23 +4952,15 @@ UnsignedDivisionByConstantInfo::get(Divisor); unsigned PreShift = 0, PostShift = 0; - // If the divisor is even, we can avoid using the expensive fixup by - // shifting the divided value upfront. - if (magics.IsAdd && !Divisor[0]) { - PreShift = Divisor.countTrailingZeros(); - // Get magic number for the shifted divisor. - magics = - UnsignedDivisionByConstantInfo::get(Divisor.lshr(PreShift), PreShift); - assert(!magics.IsAdd && "Should use cheap fixup now"); - } - unsigned SelNPQ; if (!magics.IsAdd || Divisor.isOneValue()) { assert(magics.ShiftAmount < Divisor.getBitWidth() && "We shouldn't generate an undefined shift!"); PostShift = magics.ShiftAmount; + PreShift = magics.PreShift; SelNPQ = false; } else { + assert(magics.PreShift == 0 && "Unexpected pre-shift"); PostShift = magics.ShiftAmount - 1; SelNPQ = true; } diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -6046,23 +6046,15 @@ UnsignedDivisionByConstantInfo::get(Divisor, LeadingZeros); unsigned PreShift = 0, PostShift = 0; - // If the divisor is even, we can avoid using the expensive fixup by - // shifting the divided value upfront. - if (magics.IsAdd && !Divisor[0]) { - PreShift = Divisor.countTrailingZeros(); - // Get magic number for the shifted divisor. - magics = UnsignedDivisionByConstantInfo::get(Divisor.lshr(PreShift), - PreShift + LeadingZeros); - assert(!magics.IsAdd && "Should use cheap fixup now"); - } - unsigned SelNPQ; if (!magics.IsAdd || Divisor.isOne()) { assert(magics.ShiftAmount < Divisor.getBitWidth() && "We shouldn't generate an undefined shift!"); PostShift = magics.ShiftAmount; + PreShift = magics.PreShift; SelNPQ = false; } else { + assert(magics.PreShift == 0 && "Unexpected pre-shift"); PostShift = magics.ShiftAmount - 1; SelNPQ = true; } diff --git a/llvm/lib/Support/DivisionByConstantInfo.cpp b/llvm/lib/Support/DivisionByConstantInfo.cpp --- a/llvm/lib/Support/DivisionByConstantInfo.cpp +++ b/llvm/lib/Support/DivisionByConstantInfo.cpp @@ -71,7 +71,8 @@ /// LeadingZeros can be used to simplify the calculation if the upper bits /// of the divided value are known zero. UnsignedDivisionByConstantInfo -UnsignedDivisionByConstantInfo::get(const APInt &D, unsigned LeadingZeros) { +UnsignedDivisionByConstantInfo::get(const APInt &D, unsigned LeadingZeros, + bool AllowEvenDivisorOptimization) { assert(!D.isZero() && "Precondition violation."); assert(D.getBitWidth() > 1 && "Does not work at smaller bitwidths."); @@ -129,8 +130,20 @@ Delta -= R2; } while (P < D.getBitWidth() * 2 && (Q1.ult(Delta) || (Q1 == Delta && R1.isZero()))); + + if (Retval.IsAdd && !D[0] && AllowEvenDivisorOptimization) { + unsigned PreShift = D.countTrailingZeros(); + APInt ShiftedD = D.lshr(PreShift); + Retval = + UnsignedDivisionByConstantInfo::get(ShiftedD, LeadingZeros + PreShift); + assert(Retval.IsAdd == 0 && Retval.PreShift == 0); + Retval.PreShift = PreShift; + return Retval; + } + Retval.Magic = std::move(Q2); // resulting magic number ++Retval.Magic; Retval.ShiftAmount = P - D.getBitWidth(); // resulting shift + Retval.PreShift = 0; return Retval; } diff --git a/llvm/unittests/Support/DivisionByConstantTest.cpp b/llvm/unittests/Support/DivisionByConstantTest.cpp --- a/llvm/unittests/Support/DivisionByConstantTest.cpp +++ b/llvm/unittests/Support/DivisionByConstantTest.cpp @@ -108,32 +108,23 @@ // Clip to the number of leading zeros in the divisor. LeadingZeros = std::min(LeadingZeros, Divisor.countLeadingZeros()); if (LeadingZeros > 0) { - Magics = UnsignedDivisionByConstantInfo::get(Divisor, LeadingZeros); + Magics = UnsignedDivisionByConstantInfo::get( + Divisor, LeadingZeros, AllowEvenDivisorOptimization); assert(!Magics.IsAdd && "Should use cheap fixup now"); } } unsigned PreShift = 0; - if (AllowEvenDivisorOptimization) { - // If the divisor is even, we can avoid using the expensive fixup by - // shifting the divided value upfront. - if (Magics.IsAdd && !Divisor[0]) { - PreShift = Divisor.countTrailingZeros(); - // Get magic number for the shifted divisor. - Magics = - UnsignedDivisionByConstantInfo::get(Divisor.lshr(PreShift), PreShift); - assert(!Magics.IsAdd && "Should use cheap fixup now"); - } - } - unsigned PostShift = 0; bool UseNPQ = false; if (!Magics.IsAdd || Divisor.isOne()) { assert(Magics.ShiftAmount < Divisor.getBitWidth() && "We shouldn't generate an undefined shift!"); + PreShift = Magics.PreShift; PostShift = Magics.ShiftAmount; UseNPQ = false; } else { + assert(Magics.PreShift == 0 && "Unexpected pre-shift"); PostShift = Magics.ShiftAmount - 1; assert(PostShift < Divisor.getBitWidth() && "We shouldn't generate an undefined shift!");