This is an archive of the discontinued LLVM Phabricator instance.

Differential D110807

[APInt] Stop using soft-deprecated constructors and methods in llvm. NFC.
ClosedPublic

Authored by foad on Sep 30 2021, 3:28 AM.

Details

Reviewers
jdoerfert
sstefan1
baziotis
lattner
craig.topper
Commits
rGa9bceb2b059d: [APInt] Stop using soft-deprecated constructors and methods in llvm. NFC.
Summary

Stop using APInt constructors and methods that were soft-deprecated in
D109483. This fixes all the uses I found in llvm, except for the APInt
unit tests which should still test the deprecated methods.

Diff Detail

Event Timeline

foad created this revision.Sep 30 2021, 3:28 AM
Herald added subscribers: ormris, dexonsmith, ecnelises and 10 others. · View Herald TranscriptSep 30 2021, 3:28 AM
foad requested review of this revision.Sep 30 2021, 3:28 AM
Herald added a reviewer: jdoerfert. · View Herald TranscriptSep 30 2021, 3:28 AM
Herald added a reviewer: sstefan1. · View Herald Transcript
Herald added a reviewer: baziotis. · View Herald Transcript
Herald added a project: Restricted Project. · View Herald Transcript
Herald added a subscriber: llvm-commits. · View Herald Transcript
foad added reviewers: lattner, craig.topper.Sep 30 2021, 3:29 AM
Harbormaster completed remote builds in B126514: Diff 376146.Sep 30 2021, 4:46 AM
lattner accepted this revision.Oct 3 2021, 5:28 PM

Thank you Jay!

This revision is now accepted and ready to land.Oct 3 2021, 5:28 PM
This revision was landed with ongoing or failed builds.Oct 4 2021, 12:58 AM
Closed by commit rGa9bceb2b059d: [APInt] Stop using soft-deprecated constructors and methods in llvm. NFC. (authored by foad). · Explain Why
This revision was automatically updated to reflect the committed changes.
foad added a commit: rGa9bceb2b059d: [APInt] Stop using soft-deprecated constructors and methods in llvm. NFC..

Revision Contents

PathSize
llvm/
include/
llvm/
ADT/
6 lines
CodeGen/
2 lines
IR/
2 lines
4 lines
Support/
6 lines
lib/
Analysis/
8 lines
10 lines
12 lines
3 lines
8 lines
35 lines
CodeGen/
2 lines
GlobalISel/
4 lines
2 lines
2 lines
2 lines
SelectionDAG/
12 lines
4 lines
48 lines
IR/
2 lines
4 lines
12 lines
6 lines
4 lines
4 lines
Support/
2 lines
8 lines
2 lines
Target/
AArch64/
2 lines
4 lines
ARM/
2 lines
4 lines
Hexagon/
2 lines
Mips/
4 lines
4 lines
PowerPC/
2 lines
X86/
4 lines
81 lines
6 lines
2 lines
Transforms/
IPO/
8 lines
InstCombine/
8 lines
18 lines
2 lines
10 lines
98 lines
6 lines
19 lines
19 lines
2 lines
Scalar/
7 lines
2 lines
13 lines
2 lines
Utils/
2 lines
unittests/
IR/
10 lines
16 lines
Support/
2 lines
utils/
TableGen/
2 lines

Diff 376819

llvm/include/llvm/ADT/APInt.h

Show First 20 Lines • Show All 177 Lines • ▼ Show 20 Linespublic:
/// NOTE: This is soft-deprecated. Please use `getZero()` instead. /// NOTE: This is soft-deprecated. Please use `getZero()` instead.
static APInt getNullValue(unsigned numBits) { return getZero(numBits); } static APInt getNullValue(unsigned numBits) { return getZero(numBits); }
/// Return an APInt zero bits wide. /// Return an APInt zero bits wide.
static APInt getZeroWidth() { return getZero(0); } static APInt getZeroWidth() { return getZero(0); }
/// Gets maximum unsigned value of APInt for specific bit width. /// Gets maximum unsigned value of APInt for specific bit width.
static APInt getMaxValue(unsigned numBits) { static APInt getMaxValue(unsigned numBits) { return getAllOnes(numBits); }
return getAllOnesValue(numBits);
}
/// Gets maximum signed value of APInt for a specific bit width. /// Gets maximum signed value of APInt for a specific bit width.
static APInt getSignedMaxValue(unsigned numBits) { static APInt getSignedMaxValue(unsigned numBits) {
APInt API = getAllOnesValue(numBits); APInt API = getAllOnes(numBits);
API.clearBit(numBits - 1); API.clearBit(numBits - 1);
return API; return API;
} }
/// Gets minimum unsigned value of APInt for a specific bit width. /// Gets minimum unsigned value of APInt for a specific bit width.
static APInt getMinValue(unsigned numBits) { return APInt(numBits, 0); } static APInt getMinValue(unsigned numBits) { return APInt(numBits, 0); }
/// Gets minimum signed value of APInt for a specific bit width. /// Gets minimum signed value of APInt for a specific bit width.
▲ Show 20 Lines • Show All 2,069 Lines • Show Last 20 Lines

llvm/include/llvm/CodeGen/BasicTTIImpl.h

Show First 20 Lines • Show All 1,232 Lines • ▼ Show 20 Lines if (VecTySize > VecTyLTSize) {
// will be used. // will be used.
Cost *= UsedInsts.count() / NumLegalInsts; Cost *= UsedInsts.count() / NumLegalInsts;
} }
// Then plus the cost of interleave operation. // Then plus the cost of interleave operation.
assert(Indices.size() <= Factor && assert(Indices.size() <= Factor &&
"Interleaved memory op has too many members"); "Interleaved memory op has too many members");
APInt DemandedLoadStoreElts = APInt::getNullValue(NumElts); APInt DemandedLoadStoreElts = APInt::getZero(NumElts);
for (unsigned Index : Indices) { for (unsigned Index : Indices) {
assert(Index < Factor && "Invalid index for interleaved memory op"); assert(Index < Factor && "Invalid index for interleaved memory op");
for (unsigned Elm = 0; Elm < NumSubElts; Elm++) for (unsigned Elm = 0; Elm < NumSubElts; Elm++)
DemandedLoadStoreElts.setBit(Index + Elm * Factor); DemandedLoadStoreElts.setBit(Index + Elm * Factor);
} }
if (Opcode == Instruction::Load) { if (Opcode == Instruction::Load) {
// The interleave cost is similar to extract sub vectors' elements // The interleave cost is similar to extract sub vectors' elements
▲ Show 20 Lines • Show All 964 Lines • Show Last 20 Lines

llvm/include/llvm/IR/Constants.h

Show First 20 Lines • Show All 197 Lines • ▼ Show 20 Linespublic:
/// potential for an assertion from getZExtValue(). /// potential for an assertion from getZExtValue().
/// Determine if the value is one. /// Determine if the value is one.
bool isOne() const { return Val.isOne(); } bool isOne() const { return Val.isOne(); }
/// This function will return true iff every bit in this constant is set /// This function will return true iff every bit in this constant is set
/// to true. /// to true.
/// @returns true iff this constant's bits are all set to true. /// @returns true iff this constant's bits are all set to true.
/// Determine if the value is all ones. /// Determine if the value is all ones.
bool isMinusOne() const { return Val.isAllOnesValue(); } bool isMinusOne() const { return Val.isAllOnes(); }
/// This function will return true iff this constant represents the largest /// This function will return true iff this constant represents the largest
/// value that may be represented by the constant's type. /// value that may be represented by the constant's type.
/// @returns true iff this is the largest value that may be represented /// @returns true iff this is the largest value that may be represented
/// by this type. /// by this type.
/// Determine if the value is maximal. /// Determine if the value is maximal.
bool isMaxValue(bool IsSigned) const { bool isMaxValue(bool IsSigned) const {
if (IsSigned) if (IsSigned)
▲ Show 20 Lines • Show All 1,218 Lines • Show Last 20 Lines

llvm/include/llvm/IR/PatternMatch.h

Show First 20 Lines • Show All 432 Lines • ▼ Show 20 Lines
}; };
/// Match an integer or vector with any integral constant. /// Match an integer or vector with any integral constant.
/// For vectors, this includes constants with undefined elements. /// For vectors, this includes constants with undefined elements.
inline cst_pred_ty<is_any_apint> m_AnyIntegralConstant() { inline cst_pred_ty<is_any_apint> m_AnyIntegralConstant() {
return cst_pred_ty<is_any_apint>(); return cst_pred_ty<is_any_apint>();
} }
struct is_all_ones { struct is_all_ones {
bool isValue(const APInt &C) { return C.isAllOnesValue(); } bool isValue(const APInt &C) { return C.isAllOnes(); }
}; };
/// Match an integer or vector with all bits set. /// Match an integer or vector with all bits set.
/// For vectors, this includes constants with undefined elements. /// For vectors, this includes constants with undefined elements.
inline cst_pred_ty<is_all_ones> m_AllOnes() { inline cst_pred_ty<is_all_ones> m_AllOnes() {
return cst_pred_ty<is_all_ones>(); return cst_pred_ty<is_all_ones>();
} }
struct is_maxsignedvalue { struct is_maxsignedvalue {
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines
/// Match an integer or vector of non-positive values. /// Match an integer or vector of non-positive values.
/// For vectors, this includes constants with undefined elements. /// For vectors, this includes constants with undefined elements.
inline cst_pred_ty<is_nonpositive> m_NonPositive() { inline cst_pred_ty<is_nonpositive> m_NonPositive() {
return cst_pred_ty<is_nonpositive>(); return cst_pred_ty<is_nonpositive>();
} }
inline api_pred_ty<is_nonpositive> m_NonPositive(const APInt *&V) { return V; } inline api_pred_ty<is_nonpositive> m_NonPositive(const APInt *&V) { return V; }
struct is_one { struct is_one {
bool isValue(const APInt &C) { return C.isOneValue(); } bool isValue(const APInt &C) { return C.isOne(); }
}; };
/// Match an integer 1 or a vector with all elements equal to 1. /// Match an integer 1 or a vector with all elements equal to 1.
/// For vectors, this includes constants with undefined elements. /// For vectors, this includes constants with undefined elements.
inline cst_pred_ty<is_one> m_One() { inline cst_pred_ty<is_one> m_One() {
return cst_pred_ty<is_one>(); return cst_pred_ty<is_one>();
} }
struct is_zero_int { struct is_zero_int {
▲ Show 20 Lines • Show All 2,005 Lines • Show Last 20 Lines

llvm/include/llvm/Support/KnownBits.h

Show First 20 Lines • Show All 65 Lines • ▼ Show 20 Linespublic:
void resetAll() { void resetAll() {
Zero.clearAllBits(); Zero.clearAllBits();
One.clearAllBits(); One.clearAllBits();
} }
/// Returns true if value is all zero. /// Returns true if value is all zero.
bool isZero() const { bool isZero() const {
assert(!hasConflict() && "KnownBits conflict!"); assert(!hasConflict() && "KnownBits conflict!");
return Zero.isAllOnesValue(); return Zero.isAllOnes();
} }
/// Returns true if value is all one bits. /// Returns true if value is all one bits.
bool isAllOnes() const { bool isAllOnes() const {
assert(!hasConflict() && "KnownBits conflict!"); assert(!hasConflict() && "KnownBits conflict!");
return One.isAllOnesValue(); return One.isAllOnes();
} }
/// Make all bits known to be zero and discard any previous information. /// Make all bits known to be zero and discard any previous information.
void setAllZero() { void setAllZero() {
Zero.setAllBits(); Zero.setAllBits();
One.clearAllBits(); One.clearAllBits();
} }
▲ Show 20 Lines • Show All 200 Lines • ▼ Show 20 Linespublic:
/// Compute known bits common to LHS and RHS. /// Compute known bits common to LHS and RHS.
static KnownBits commonBits(const KnownBits &LHS, const KnownBits &RHS) { static KnownBits commonBits(const KnownBits &LHS, const KnownBits &RHS) {
return KnownBits(LHS.Zero & RHS.Zero, LHS.One & RHS.One); return KnownBits(LHS.Zero & RHS.Zero, LHS.One & RHS.One);
} }
/// Return true if LHS and RHS have no common bits set. /// Return true if LHS and RHS have no common bits set.
static bool haveNoCommonBitsSet(const KnownBits &LHS, const KnownBits &RHS) { static bool haveNoCommonBitsSet(const KnownBits &LHS, const KnownBits &RHS) {
return (LHS.Zero | RHS.Zero).isAllOnesValue(); return (LHS.Zero | RHS.Zero).isAllOnes();
} }
/// Compute known bits resulting from adding LHS, RHS and a 1-bit Carry. /// Compute known bits resulting from adding LHS, RHS and a 1-bit Carry.
static KnownBits computeForAddCarry( static KnownBits computeForAddCarry(
const KnownBits &LHS, const KnownBits &RHS, const KnownBits &Carry); const KnownBits &LHS, const KnownBits &RHS, const KnownBits &Carry);
/// Compute known bits resulting from adding LHS and RHS. /// Compute known bits resulting from adding LHS and RHS.
static KnownBits computeForAddSub(bool Add, bool NSW, const KnownBits &LHS, static KnownBits computeForAddSub(bool Add, bool NSW, const KnownBits &LHS,
▲ Show 20 Lines • Show All 132 Lines • Show Last 20 Lines

llvm/lib/Analysis/CmpInstAnalysis.cpp

Show First 20 Lines • Show All 71 Lines • ▼ Show 20 Linesbool llvm::decomposeBitTestICmp(Value *LHS, Value *RHS,
if (!match(RHS, m_APInt(C))) if (!match(RHS, m_APInt(C)))
return false; return false;
switch (Pred) { switch (Pred) {
default: default:
return false; return false;
case ICmpInst::ICMP_SLT: case ICmpInst::ICMP_SLT:
// X < 0 is equivalent to (X & SignMask) != 0. // X < 0 is equivalent to (X & SignMask) != 0.
if (!C->isNullValue()) if (!C->isZero())
return false; return false;
Mask = APInt::getSignMask(C->getBitWidth()); Mask = APInt::getSignMask(C->getBitWidth());
Pred = ICmpInst::ICMP_NE; Pred = ICmpInst::ICMP_NE;
break; break;
case ICmpInst::ICMP_SLE: case ICmpInst::ICMP_SLE:
// X <= -1 is equivalent to (X & SignMask) != 0. // X <= -1 is equivalent to (X & SignMask) != 0.
if (!C->isAllOnesValue()) if (!C->isAllOnes())
return false; return false;
Mask = APInt::getSignMask(C->getBitWidth()); Mask = APInt::getSignMask(C->getBitWidth());
Pred = ICmpInst::ICMP_NE; Pred = ICmpInst::ICMP_NE;
break; break;
case ICmpInst::ICMP_SGT: case ICmpInst::ICMP_SGT:
// X > -1 is equivalent to (X & SignMask) == 0. // X > -1 is equivalent to (X & SignMask) == 0.
if (!C->isAllOnesValue()) if (!C->isAllOnes())
return false; return false;
Mask = APInt::getSignMask(C->getBitWidth()); Mask = APInt::getSignMask(C->getBitWidth());
Pred = ICmpInst::ICMP_EQ; Pred = ICmpInst::ICMP_EQ;
break; break;
case ICmpInst::ICMP_SGE: case ICmpInst::ICMP_SGE:
// X >= 0 is equivalent to (X & SignMask) == 0. // X >= 0 is equivalent to (X & SignMask) == 0.
if (!C->isNullValue()) if (!C->isZero())
return false; return false;
Mask = APInt::getSignMask(C->getBitWidth()); Mask = APInt::getSignMask(C->getBitWidth());
Pred = ICmpInst::ICMP_EQ; Pred = ICmpInst::ICMP_EQ;
break; break;
case ICmpInst::ICMP_ULT: case ICmpInst::ICMP_ULT:
// X <u 2^n is equivalent to (X & ~(2^n-1)) == 0. // X <u 2^n is equivalent to (X & ~(2^n-1)) == 0.
if (!C->isPowerOf2()) if (!C->isPowerOf2())
return false; return false;
Show All 34 Lines

llvm/lib/Analysis/ConstantFolding.cpp

Show First 20 Lines • Show All 789 Lines • ▼ Show 20 LinesConstant *SymbolicallyEvaluateBinop(unsigned Opc, Constant *Op0, Constant *Op1,
// Fold (and 0xffffffff00000000, (shl x, 32)) -> shl. // Fold (and 0xffffffff00000000, (shl x, 32)) -> shl.
// Fold (lshr (or X, Y), 32) -> (lshr [X/Y], 32) if one doesn't contribute // Fold (lshr (or X, Y), 32) -> (lshr [X/Y], 32) if one doesn't contribute
// bits. // bits.
if (Opc == Instruction::And) { if (Opc == Instruction::And) {
KnownBits Known0 = computeKnownBits(Op0, DL); KnownBits Known0 = computeKnownBits(Op0, DL);
KnownBits Known1 = computeKnownBits(Op1, DL); KnownBits Known1 = computeKnownBits(Op1, DL);
if ((Known1.One | Known0.Zero).isAllOnesValue()) { if ((Known1.One | Known0.Zero).isAllOnes()) {
// All the bits of Op0 that the 'and' could be masking are already zero. // All the bits of Op0 that the 'and' could be masking are already zero.
return Op0; return Op0;
} }
if ((Known0.One | Known1.Zero).isAllOnesValue()) { if ((Known0.One | Known1.Zero).isAllOnes()) {
// All the bits of Op1 that the 'and' could be masking are already zero. // All the bits of Op1 that the 'and' could be masking are already zero.
return Op1; return Op1;
} }
Known0 &= Known1; Known0 &= Known1;
if (Known0.isConstant()) if (Known0.isConstant())
return ConstantInt::get(Op0->getType(), Known0.getConstant()); return ConstantInt::get(Op0->getType(), Known0.getConstant());
} }
▲ Show 20 Lines • Show All 1,835 Lines • ▼ Show 20 Lines case Intrinsic::ssub_sat:
return ConstantInt::get(Ty, C0->usub_sat(*C1)); return ConstantInt::get(Ty, C0->usub_sat(*C1));
else else
return ConstantInt::get(Ty, C0->ssub_sat(*C1)); return ConstantInt::get(Ty, C0->ssub_sat(*C1));
case Intrinsic::cttz: case Intrinsic::cttz:
case Intrinsic::ctlz: case Intrinsic::ctlz:
assert(C1 && "Must be constant int"); assert(C1 && "Must be constant int");
// cttz(0, 1) and ctlz(0, 1) are undef. // cttz(0, 1) and ctlz(0, 1) are undef.
if (C1->isOneValue() && (!C0 || C0->isNullValue())) if (C1->isOne() && (!C0 || C0->isZero()))
return UndefValue::get(Ty); return UndefValue::get(Ty);
if (!C0) if (!C0)
return Constant::getNullValue(Ty); return Constant::getNullValue(Ty);
if (IntrinsicID == Intrinsic::cttz) if (IntrinsicID == Intrinsic::cttz)
return ConstantInt::get(Ty, C0->countTrailingZeros()); return ConstantInt::get(Ty, C0->countTrailingZeros());
else else
return ConstantInt::get(Ty, C0->countLeadingZeros()); return ConstantInt::get(Ty, C0->countLeadingZeros());
case Intrinsic::abs: case Intrinsic::abs:
// Undef or minimum val operand with poison min --> undef // Undef or minimum val operand with poison min --> undef
assert(C1 && "Must be constant int"); assert(C1 && "Must be constant int");
if (C1->isOneValue() && (!C0 || C0->isMinSignedValue())) if (C1->isOne() && (!C0 || C0->isMinSignedValue()))
return UndefValue::get(Ty); return UndefValue::get(Ty);
// Undef operand with no poison min --> 0 (sign bit must be clear) // Undef operand with no poison min --> 0 (sign bit must be clear)
if (C1->isNullValue() && !C0) if (C1->isZero() && !C0)
return Constant::getNullValue(Ty); return Constant::getNullValue(Ty);
return ConstantInt::get(Ty, C0->abs()); return ConstantInt::get(Ty, C0->abs());
} }
return nullptr; return nullptr;
} }
▲ Show 20 Lines • Show All 624 Lines • Show Last 20 Lines

llvm/lib/Analysis/InstructionSimplify.cpp

Show First 20 Lines • Show All 2,047 Lines • ▼ Show 20 Linesstatic Value *SimplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
// A mask that only clears known zeros of a shifted value is a no-op. // A mask that only clears known zeros of a shifted value is a no-op.
Value *X; Value *X;
const APInt *Mask; const APInt *Mask;
const APInt *ShAmt; const APInt *ShAmt;
if (match(Op1, m_APInt(Mask))) { if (match(Op1, m_APInt(Mask))) {
// If all bits in the inverted and shifted mask are clear: // If all bits in the inverted and shifted mask are clear:
// and (shl X, ShAmt), Mask --> shl X, ShAmt // and (shl X, ShAmt), Mask --> shl X, ShAmt
if (match(Op0, m_Shl(m_Value(X), m_APInt(ShAmt))) && if (match(Op0, m_Shl(m_Value(X), m_APInt(ShAmt))) &&
(~(*Mask)).lshr(*ShAmt).isNullValue()) (~(*Mask)).lshr(*ShAmt).isZero())
return Op0; return Op0;
// If all bits in the inverted and shifted mask are clear: // If all bits in the inverted and shifted mask are clear:
// and (lshr X, ShAmt), Mask --> lshr X, ShAmt // and (lshr X, ShAmt), Mask --> lshr X, ShAmt
if (match(Op0, m_LShr(m_Value(X), m_APInt(ShAmt))) && if (match(Op0, m_LShr(m_Value(X), m_APInt(ShAmt))) &&
(~(*Mask)).shl(*ShAmt).isNullValue()) (~(*Mask)).shl(*ShAmt).isZero())
return Op0; return Op0;
} }
// If we have a multiplication overflow check that is being 'and'ed with a // If we have a multiplication overflow check that is being 'and'ed with a
// check that one of the multipliers is not zero, we can omit the 'and', and // check that one of the multipliers is not zero, we can omit the 'and', and
// only keep the overflow check. // only keep the overflow check.
if (isCheckForZeroAndMulWithOverflow(Op0, Op1, true)) if (isCheckForZeroAndMulWithOverflow(Op0, Op1, true))
return Op1; return Op1;
▲ Show 20 Lines • Show All 1,033 Lines • ▼ Show 20 Lines if (match(LHS, m_Shl(m_Power2(), m_Value())) &&
// //
// We know it is safe if: // We know it is safe if:
// - The shift is nsw. We can't shift out the one bit. // - The shift is nsw. We can't shift out the one bit.
// - The shift is nuw. We can't shift out the one bit. // - The shift is nuw. We can't shift out the one bit.
// - C2 is one. // - C2 is one.
// - C isn't zero. // - C isn't zero.
if (Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(LBO)) || if (Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(LBO)) ||
Q.IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(LBO)) || Q.IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(LBO)) ||
match(LHS, m_Shl(m_One(), m_Value())) || !C->isNullValue()) { match(LHS, m_Shl(m_One(), m_Value())) || !C->isZero()) {
if (Pred == ICmpInst::ICMP_EQ) if (Pred == ICmpInst::ICMP_EQ)
return ConstantInt::getFalse(GetCompareTy(RHS)); return ConstantInt::getFalse(GetCompareTy(RHS));
if (Pred == ICmpInst::ICMP_NE) if (Pred == ICmpInst::ICMP_NE)
return ConstantInt::getTrue(GetCompareTy(RHS)); return ConstantInt::getTrue(GetCompareTy(RHS));
} }
} }
// TODO: This is overly constrained. LHS can be any power-of-2. // TODO: This is overly constrained. LHS can be any power-of-2.
▲ Show 20 Lines • Show All 1,306 Lines • ▼ Show 20 Lines if (Q.DL.getTypeSizeInBits(Ops.back()->getType()) == IdxWidth) {
// Avoid creating inttoptr of zero here: While LLVMs treatment of // Avoid creating inttoptr of zero here: While LLVMs treatment of
// inttoptr is generally conservative, this particular case is folded to // inttoptr is generally conservative, this particular case is folded to
// a null pointer, which will have incorrect provenance. // a null pointer, which will have incorrect provenance.
// gep (gep V, C), (sub 0, V) -> C // gep (gep V, C), (sub 0, V) -> C
if (match(Ops.back(), if (match(Ops.back(),
m_Sub(m_Zero(), m_PtrToInt(m_Specific(StrippedBasePtr)))) && m_Sub(m_Zero(), m_PtrToInt(m_Specific(StrippedBasePtr)))) &&
!BasePtrOffset.isNullValue()) { !BasePtrOffset.isZero()) {
auto *CI = ConstantInt::get(GEPTy->getContext(), BasePtrOffset); auto *CI = ConstantInt::get(GEPTy->getContext(), BasePtrOffset);
return ConstantExpr::getIntToPtr(CI, GEPTy); return ConstantExpr::getIntToPtr(CI, GEPTy);
} }
// gep (gep V, C), (xor V, -1) -> C-1 // gep (gep V, C), (xor V, -1) -> C-1
if (match(Ops.back(), if (match(Ops.back(),
m_Xor(m_PtrToInt(m_Specific(StrippedBasePtr)), m_AllOnes())) && m_Xor(m_PtrToInt(m_Specific(StrippedBasePtr)), m_AllOnes())) &&
!BasePtrOffset.isOneValue()) { !BasePtrOffset.isOne()) {
auto *CI = ConstantInt::get(GEPTy->getContext(), BasePtrOffset - 1); auto *CI = ConstantInt::get(GEPTy->getContext(), BasePtrOffset - 1);
return ConstantExpr::getIntToPtr(CI, GEPTy); return ConstantExpr::getIntToPtr(CI, GEPTy);
} }
} }
} }
// Check to see if this is constant foldable. // Check to see if this is constant foldable.
if (!all_of(Ops, [](Value *V) { return isa<Constant>(V); })) if (!all_of(Ops, [](Value *V) { return isa<Constant>(V); }))
▲ Show 20 Lines • Show All 1,416 Lines • ▼ Show 20 Lines case Intrinsic::fshr: {
// If shift amount is undef, assume it is zero. // If shift amount is undef, assume it is zero.
if (Q.isUndefValue(ShAmtArg)) if (Q.isUndefValue(ShAmtArg))
return Call->getArgOperand(IID == Intrinsic::fshl ? 0 : 1); return Call->getArgOperand(IID == Intrinsic::fshl ? 0 : 1);
const APInt *ShAmtC; const APInt *ShAmtC;
if (match(ShAmtArg, m_APInt(ShAmtC))) { if (match(ShAmtArg, m_APInt(ShAmtC))) {
// If there's effectively no shift, return the 1st arg or 2nd arg. // If there's effectively no shift, return the 1st arg or 2nd arg.
APInt BitWidth = APInt(ShAmtC->getBitWidth(), ShAmtC->getBitWidth()); APInt BitWidth = APInt(ShAmtC->getBitWidth(), ShAmtC->getBitWidth());
if (ShAmtC->urem(BitWidth).isNullValue()) if (ShAmtC->urem(BitWidth).isZero())
return Call->getArgOperand(IID == Intrinsic::fshl ? 0 : 1); return Call->getArgOperand(IID == Intrinsic::fshl ? 0 : 1);
} }
// Rotating zero by anything is zero. // Rotating zero by anything is zero.
if (match(Op0, m_Zero()) && match(Op1, m_Zero())) if (match(Op0, m_Zero()) && match(Op1, m_Zero()))
return ConstantInt::getNullValue(F->getReturnType()); return ConstantInt::getNullValue(F->getReturnType());
// Rotating -1 by anything is -1. // Rotating -1 by anything is -1.
▲ Show 20 Lines • Show All 526 Lines • Show Last 20 Lines

llvm/lib/Analysis/LazyValueInfo.cpp

Show First 20 Lines • Show All 1,111 Lines • ▼ Show 20 Lines if (EdgePred == ICmpInst::ICMP_EQ) {
KnownBits Known; KnownBits Known;
Known.Zero = ~*C & *Mask; Known.Zero = ~*C & *Mask;
Known.One = *C & *Mask; Known.One = *C & *Mask;
return ValueLatticeElement::getRange( return ValueLatticeElement::getRange(
ConstantRange::fromKnownBits(Known, /*IsSigned*/ false)); ConstantRange::fromKnownBits(Known, /*IsSigned*/ false));
} }
// If (Val & Mask) != 0 then the value must be larger than the lowest set // If (Val & Mask) != 0 then the value must be larger than the lowest set
// bit of Mask. // bit of Mask.
if (EdgePred == ICmpInst::ICMP_NE && !Mask->isNullValue() && if (EdgePred == ICmpInst::ICMP_NE && !Mask->isZero() && C->isZero()) {
C->isNullValue()) {
unsigned BitWidth = Ty->getIntegerBitWidth(); unsigned BitWidth = Ty->getIntegerBitWidth();
return ValueLatticeElement::getRange(ConstantRange::getNonEmpty( return ValueLatticeElement::getRange(ConstantRange::getNonEmpty(
APInt::getOneBitSet(BitWidth, Mask->countTrailingZeros()), APInt::getOneBitSet(BitWidth, Mask->countTrailingZeros()),
APInt::getZero(BitWidth))); APInt::getZero(BitWidth)));
} }
} }
return ValueLatticeElement::getOverdefined(); return ValueLatticeElement::getOverdefined();
▲ Show 20 Lines • Show All 837 Lines • Show Last 20 Lines

llvm/lib/Analysis/ScalarEvolution.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 6,138 Lines • ▼ Show 20 Lines return setRange(Trunc, SignHint,
RangeType)); RangeType));
} }
if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) { if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) {
// If there's no unsigned wrap, the value will never be less than its // If there's no unsigned wrap, the value will never be less than its
// initial value. // initial value.
if (AddRec->hasNoUnsignedWrap()) { if (AddRec->hasNoUnsignedWrap()) {
APInt UnsignedMinValue = getUnsignedRangeMin(AddRec->getStart()); APInt UnsignedMinValue = getUnsignedRangeMin(AddRec->getStart());
if (!UnsignedMinValue.isNullValue()) if (!UnsignedMinValue.isZero())
ConservativeResult = ConservativeResult.intersectWith( ConservativeResult = ConservativeResult.intersectWith(
ConstantRange(UnsignedMinValue, APInt(BitWidth, 0)), RangeType); ConstantRange(UnsignedMinValue, APInt(BitWidth, 0)), RangeType);
} }
// If there's no signed wrap, and all the operands except initial value have // If there's no signed wrap, and all the operands except initial value have
// the same sign or zero, the value won't ever be: // the same sign or zero, the value won't ever be:
// 1: smaller than initial value if operands are non negative, // 1: smaller than initial value if operands are non negative,
// 2: bigger than initial value if operands are non positive. // 2: bigger than initial value if operands are non positive.
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines if (U->getType()->isPointerTy()) {
unsigned ptrSize = DL.getPointerTypeSizeInBits(U->getType()); unsigned ptrSize = DL.getPointerTypeSizeInBits(U->getType());
int ptrIdxDiff = ptrSize - BitWidth; int ptrIdxDiff = ptrSize - BitWidth;
if (ptrIdxDiff > 0 && ptrSize > BitWidth && NS > (unsigned)ptrIdxDiff) if (ptrIdxDiff > 0 && ptrSize > BitWidth && NS > (unsigned)ptrIdxDiff)
NS -= ptrIdxDiff; NS -= ptrIdxDiff;
} }
if (NS > 1) { if (NS > 1) {
// If we know any of the sign bits, we know all of the sign bits. // If we know any of the sign bits, we know all of the sign bits.
if (!Known.Zero.getHiBits(NS).isNullValue()) if (!Known.Zero.getHiBits(NS).isZero())
Known.Zero.setHighBits(NS); Known.Zero.setHighBits(NS);
if (!Known.One.getHiBits(NS).isNullValue()) if (!Known.One.getHiBits(NS).isZero())
Known.One.setHighBits(NS); Known.One.setHighBits(NS);
} }
if (Known.getMinValue() != Known.getMaxValue() + 1) if (Known.getMinValue() != Known.getMaxValue() + 1)
ConservativeResult = ConservativeResult.intersectWith( ConservativeResult = ConservativeResult.intersectWith(
ConstantRange(Known.getMinValue(), Known.getMaxValue() + 1), ConstantRange(Known.getMinValue(), Known.getMaxValue() + 1),
RangeType); RangeType);
if (NS > 1) if (NS > 1)
▲ Show 20 Lines • Show All 2,965 Lines • ▼ Show 20 LinesGetQuadraticEquation(const SCEVAddRecExpr *AddRec) {
if (!LC || !MC || !NC) { if (!LC || !MC || !NC) {
LLVM_DEBUG(dbgs() << __func__ << ": coefficients are not constant\n"); LLVM_DEBUG(dbgs() << __func__ << ": coefficients are not constant\n");
return None; return None;
} }
APInt L = LC->getAPInt(); APInt L = LC->getAPInt();
APInt M = MC->getAPInt(); APInt M = MC->getAPInt();
APInt N = NC->getAPInt(); APInt N = NC->getAPInt();
assert(!N.isNullValue() && "This is not a quadratic addrec"); assert(!N.isZero() && "This is not a quadratic addrec");
unsigned BitWidth = LC->getAPInt().getBitWidth(); unsigned BitWidth = LC->getAPInt().getBitWidth();
unsigned NewWidth = BitWidth + 1; unsigned NewWidth = BitWidth + 1;
LLVM_DEBUG(dbgs() << __func__ << ": addrec coeff bw: " LLVM_DEBUG(dbgs() << __func__ << ": addrec coeff bw: "
<< BitWidth << '\n'); << BitWidth << '\n');
// The sign-extension (as opposed to a zero-extension) here matches the // The sign-extension (as opposed to a zero-extension) here matches the
// extension used in SolveQuadraticEquationWrap (with the same motivation). // extension used in SolveQuadraticEquationWrap (with the same motivation).
N = N.sext(NewWidth); N = N.sext(NewWidth);
▲ Show 20 Lines • Show All 4,498 Lines • Show Last 20 Lines

llvm/lib/Analysis/ValueTracking.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 160 Lines • ▼ Show 20 Linesstatic bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
// cannot check their values // cannot check their values
if (isa<ScalableVectorType>(Shuf->getType())) if (isa<ScalableVectorType>(Shuf->getType()))
return false; return false;
int NumElts = int NumElts =
cast<FixedVectorType>(Shuf->getOperand(0)->getType())->getNumElements(); cast<FixedVectorType>(Shuf->getOperand(0)->getType())->getNumElements();
int NumMaskElts = cast<FixedVectorType>(Shuf->getType())->getNumElements(); int NumMaskElts = cast<FixedVectorType>(Shuf->getType())->getNumElements();
DemandedLHS = DemandedRHS = APInt::getZero(NumElts); DemandedLHS = DemandedRHS = APInt::getZero(NumElts);
if (DemandedElts.isNullValue()) if (DemandedElts.isZero())
return true; return true;
// Simple case of a shuffle with zeroinitializer. // Simple case of a shuffle with zeroinitializer.
if (all_of(Shuf->getShuffleMask(), [](int Elt) { return Elt == 0; })) { if (all_of(Shuf->getShuffleMask(), [](int Elt) { return Elt == 0; })) {
DemandedLHS.setBit(0); DemandedLHS.setBit(0);
return true; return true;
} }
for (int i = 0; i != NumMaskElts; ++i) { for (int i = 0; i != NumMaskElts; ++i) {
if (!DemandedElts[i]) if (!DemandedElts[i])
▲ Show 20 Lines • Show All 1,195 Lines • ▼ Show 20 Lines for (unsigned i = 1, e = I->getNumOperands(); i != e; ++i, ++GTI) {
// to the width of the pointer. // to the width of the pointer.
IndexBits = IndexBits.sextOrTrunc(BitWidth); IndexBits = IndexBits.sextOrTrunc(BitWidth);
// Note that inbounds does *not* guarantee nsw for the addition, as only // Note that inbounds does *not* guarantee nsw for the addition, as only
// the offset is signed, while the base address is unsigned. // the offset is signed, while the base address is unsigned.
Known = KnownBits::computeForAddSub( Known = KnownBits::computeForAddSub(
/*Add=*/true, /*NSW=*/false, Known, IndexBits); /*Add=*/true, /*NSW=*/false, Known, IndexBits);
} }
if (!Known.isUnknown() && !AccConstIndices.isNullValue()) { if (!Known.isUnknown() && !AccConstIndices.isZero()) {
KnownBits Index = KnownBits::makeConstant(AccConstIndices); KnownBits Index = KnownBits::makeConstant(AccConstIndices);
Known = KnownBits::computeForAddSub( Known = KnownBits::computeForAddSub(
/*Add=*/true, /*NSW=*/false, Known, Index); /*Add=*/true, /*NSW=*/false, Known, Index);
} }
break; break;
} }
case Instruction::PHI: { case Instruction::PHI: {
const PHINode *P = cast<PHINode>(I); const PHINode *P = cast<PHINode>(I);
▲ Show 20 Lines • Show All 875 Lines • ▼ Show 20 Lines
/// Try to detect a recurrence that monotonically increases/decreases from a /// Try to detect a recurrence that monotonically increases/decreases from a
/// non-zero starting value. These are common as induction variables. /// non-zero starting value. These are common as induction variables.
static bool isNonZeroRecurrence(const PHINode *PN) { static bool isNonZeroRecurrence(const PHINode *PN) {
BinaryOperator *BO = nullptr; BinaryOperator *BO = nullptr;
Value *Start = nullptr, *Step = nullptr; Value *Start = nullptr, *Step = nullptr;
const APInt *StartC, *StepC; const APInt *StartC, *StepC;
if (!matchSimpleRecurrence(PN, BO, Start, Step) || if (!matchSimpleRecurrence(PN, BO, Start, Step) ||
!match(Start, m_APInt(StartC)) || StartC->isNullValue()) !match(Start, m_APInt(StartC)) || StartC->isZero())
return false; return false;
switch (BO->getOpcode()) { switch (BO->getOpcode()) {
case Instruction::Add: case Instruction::Add:
// Starting from non-zero and stepping away from zero can never wrap back // Starting from non-zero and stepping away from zero can never wrap back
// to zero. // to zero.
return BO->hasNoUnsignedWrap() || return BO->hasNoUnsignedWrap() ||
(BO->hasNoSignedWrap() && match(Step, m_APInt(StepC)) && (BO->hasNoSignedWrap() && match(Step, m_APInt(StepC)) &&
StartC->isNegative() == StepC->isNegative()); StartC->isNegative() == StepC->isNegative());
case Instruction::Mul: case Instruction::Mul:
return (BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap()) && return (BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap()) &&
match(Step, m_APInt(StepC)) && !StepC->isNullValue(); match(Step, m_APInt(StepC)) && !StepC->isZero();
case Instruction::Shl: case Instruction::Shl:
return BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap(); return BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap();
case Instruction::AShr: case Instruction::AShr:
case Instruction::LShr: case Instruction::LShr:
return BO->isExact(); return BO->isExact();
default: default:
return false; return false;
} }
▲ Show 20 Lines • Show All 417 Lines • ▼ Show 20 Lines
/// Return true if V2 == V1 * C, where V1 is known non-zero, C is not 0/1 and /// Return true if V2 == V1 * C, where V1 is known non-zero, C is not 0/1 and
/// the multiplication is nuw or nsw. /// the multiplication is nuw or nsw.
static bool isNonEqualMul(const Value *V1, const Value *V2, unsigned Depth, static bool isNonEqualMul(const Value *V1, const Value *V2, unsigned Depth,
const Query &Q) { const Query &Q) {
if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(V2)) { if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(V2)) {
const APInt *C; const APInt *C;
return match(OBO, m_Mul(m_Specific(V1), m_APInt(C))) && return match(OBO, m_Mul(m_Specific(V1), m_APInt(C))) &&
(OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) && (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
!C->isNullValue() && !C->isOneValue() && !C->isZero() && !C->isOne() && isKnownNonZero(V1, Depth + 1, Q);
isKnownNonZero(V1, Depth + 1, Q);
} }
return false; return false;
} }
/// Return true if V2 == V1 << C, where V1 is known non-zero, C is not 0 and /// Return true if V2 == V1 << C, where V1 is known non-zero, C is not 0 and
/// the shift is nuw or nsw. /// the shift is nuw or nsw.
static bool isNonEqualShl(const Value *V1, const Value *V2, unsigned Depth, static bool isNonEqualShl(const Value *V1, const Value *V2, unsigned Depth,
const Query &Q) { const Query &Q) {
if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(V2)) { if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(V2)) {
const APInt *C; const APInt *C;
return match(OBO, m_Shl(m_Specific(V1), m_APInt(C))) && return match(OBO, m_Shl(m_Specific(V1), m_APInt(C))) &&
(OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) && (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
!C->isNullValue() && isKnownNonZero(V1, Depth + 1, Q); !C->isZero() && isKnownNonZero(V1, Depth + 1, Q);
} }
return false; return false;
} }
static bool isNonEqualPHIs(const PHINode *PN1, const PHINode *PN2, static bool isNonEqualPHIs(const PHINode *PN1, const PHINode *PN2,
unsigned Depth, const Query &Q) { unsigned Depth, const Query &Q) {
// Check two PHIs are in same block. // Check two PHIs are in same block.
if (PN1->getParent() != PN2->getParent()) if (PN1->getParent() != PN2->getParent())
▲ Show 20 Lines • Show All 326 Lines • ▼ Show 20 Lines case Instruction::Add:
// Special case decrementing a value (ADD X, -1): // Special case decrementing a value (ADD X, -1):
if (const auto *CRHS = dyn_cast<Constant>(U->getOperand(1))) if (const auto *CRHS = dyn_cast<Constant>(U->getOperand(1)))
if (CRHS->isAllOnesValue()) { if (CRHS->isAllOnesValue()) {
KnownBits Known(TyBits); KnownBits Known(TyBits);
computeKnownBits(U->getOperand(0), Known, Depth + 1, Q); computeKnownBits(U->getOperand(0), Known, Depth + 1, Q);
// If the input is known to be 0 or 1, the output is 0/-1, which is // If the input is known to be 0 or 1, the output is 0/-1, which is
// all sign bits set. // all sign bits set.
if ((Known.Zero | 1).isAllOnesValue()) if ((Known.Zero | 1).isAllOnes())
return TyBits; return TyBits;
// If we are subtracting one from a positive number, there is no carry // If we are subtracting one from a positive number, there is no carry
// out of the result. // out of the result.
if (Known.isNonNegative()) if (Known.isNonNegative())
return Tmp; return Tmp;
} }
Tmp2 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q); Tmp2 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
if (Tmp2 == 1) break; if (Tmp2 == 1) break;
return std::min(Tmp, Tmp2) - 1; return std::min(Tmp, Tmp2) - 1;
case Instruction::Sub: case Instruction::Sub:
Tmp2 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q); Tmp2 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
if (Tmp2 == 1) break; if (Tmp2 == 1) break;
// Handle NEG. // Handle NEG.
if (const auto *CLHS = dyn_cast<Constant>(U->getOperand(0))) if (const auto *CLHS = dyn_cast<Constant>(U->getOperand(0)))
if (CLHS->isNullValue()) { if (CLHS->isNullValue()) {
KnownBits Known(TyBits); KnownBits Known(TyBits);
computeKnownBits(U->getOperand(1), Known, Depth + 1, Q); computeKnownBits(U->getOperand(1), Known, Depth + 1, Q);
// If the input is known to be 0 or 1, the output is 0/-1, which is // If the input is known to be 0 or 1, the output is 0/-1, which is
// all sign bits set. // all sign bits set.
if ((Known.Zero | 1).isAllOnesValue()) if ((Known.Zero | 1).isAllOnes())
return TyBits; return TyBits;
// If the input is known to be positive (the sign bit is known clear), // If the input is known to be positive (the sign bit is known clear),
// the output of the NEG has the same number of sign bits as the // the output of the NEG has the same number of sign bits as the
// input. // input.
if (Known.isNonNegative()) if (Known.isNonNegative())
return Tmp2; return Tmp2;
▲ Show 20 Lines • Show All 1,528 Lines • ▼ Show 20 Lines case Instruction::SRem: {
// x / y is undefined if y == 0 or x == INT_MIN and y == -1 // x / y is undefined if y == 0 or x == INT_MIN and y == -1
const APInt *Numerator, *Denominator; const APInt *Numerator, *Denominator;
if (!match(Inst->getOperand(1), m_APInt(Denominator))) if (!match(Inst->getOperand(1), m_APInt(Denominator)))
return false; return false;
// We cannot hoist this division if the denominator is 0. // We cannot hoist this division if the denominator is 0.
if (*Denominator == 0) if (*Denominator == 0)
return false; return false;
// It's safe to hoist if the denominator is not 0 or -1. // It's safe to hoist if the denominator is not 0 or -1.
if (!Denominator->isAllOnesValue()) if (!Denominator->isAllOnes())
return true; return true;
// At this point we know that the denominator is -1. It is safe to hoist as // At this point we know that the denominator is -1. It is safe to hoist as
// long we know that the numerator is not INT_MIN. // long we know that the numerator is not INT_MIN.
if (match(Inst->getOperand(0), m_APInt(Numerator))) if (match(Inst->getOperand(0), m_APInt(Numerator)))
return !Numerator->isMinSignedValue(); return !Numerator->isMinSignedValue();
// The numerator *might* be MinSignedValue. // The numerator *might* be MinSignedValue.
return false; return false;
} }
▲ Show 20 Lines • Show All 1,204 Lines • ▼ Show 20 Linesstatic SelectPatternResult matchMinMax(CmpInst::Predicate Pred,
// An unsigned min/max can be written with a signed compare. // An unsigned min/max can be written with a signed compare.
const APInt *C2; const APInt *C2;
if ((CmpLHS == TrueVal && match(FalseVal, m_APInt(C2))) || if ((CmpLHS == TrueVal && match(FalseVal, m_APInt(C2))) ||
(CmpLHS == FalseVal && match(TrueVal, m_APInt(C2)))) { (CmpLHS == FalseVal && match(TrueVal, m_APInt(C2)))) {
// Is the sign bit set? // Is the sign bit set?
// (X <s 0) ? X : MAXVAL ==> (X >u MAXVAL) ? X : MAXVAL ==> UMAX // (X <s 0) ? X : MAXVAL ==> (X >u MAXVAL) ? X : MAXVAL ==> UMAX
// (X <s 0) ? MAXVAL : X ==> (X >u MAXVAL) ? MAXVAL : X ==> UMIN // (X <s 0) ? MAXVAL : X ==> (X >u MAXVAL) ? MAXVAL : X ==> UMIN
if (Pred == CmpInst::ICMP_SLT && C1->isNullValue() && if (Pred == CmpInst::ICMP_SLT && C1->isZero() && C2->isMaxSignedValue())
C2->isMaxSignedValue())
return {CmpLHS == TrueVal ? SPF_UMAX : SPF_UMIN, SPNB_NA, false}; return {CmpLHS == TrueVal ? SPF_UMAX : SPF_UMIN, SPNB_NA, false};
// Is the sign bit clear? // Is the sign bit clear?
// (X >s -1) ? MINVAL : X ==> (X <u MINVAL) ? MINVAL : X ==> UMAX // (X >s -1) ? MINVAL : X ==> (X <u MINVAL) ? MINVAL : X ==> UMAX
// (X >s -1) ? X : MINVAL ==> (X <u MINVAL) ? X : MINVAL ==> UMIN // (X >s -1) ? X : MINVAL ==> (X <u MINVAL) ? X : MINVAL ==> UMIN
if (Pred == CmpInst::ICMP_SGT && C1->isAllOnesValue() && if (Pred == CmpInst::ICMP_SGT && C1->isAllOnes() && C2->isMinSignedValue())
C2->isMinSignedValue())
return {CmpLHS == FalseVal ? SPF_UMAX : SPF_UMIN, SPNB_NA, false}; return {CmpLHS == FalseVal ? SPF_UMAX : SPF_UMIN, SPNB_NA, false};
} }
return {SPF_UNKNOWN, SPNB_NA, false}; return {SPF_UNKNOWN, SPNB_NA, false};
} }
bool llvm::isKnownNegation(const Value *X, const Value *Y, bool NeedNSW) { bool llvm::isKnownNegation(const Value *X, const Value *Y, bool NeedNSW) {
assert(X && Y && "Invalid operand"); assert(X && Y && "Invalid operand");
▲ Show 20 Lines • Show All 831 Lines • ▼ Show 20 Lines
} }
static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower, static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower,
APInt &Upper, const InstrInfoQuery &IIQ) { APInt &Upper, const InstrInfoQuery &IIQ) {
unsigned Width = Lower.getBitWidth(); unsigned Width = Lower.getBitWidth();
const APInt *C; const APInt *C;
switch (BO.getOpcode()) { switch (BO.getOpcode()) {
case Instruction::Add: case Instruction::Add:
if (match(BO.getOperand(1), m_APInt(C)) && !C->isNullValue()) { if (match(BO.getOperand(1), m_APInt(C)) && !C->isZero()) {
// FIXME: If we have both nuw and nsw, we should reduce the range further. // FIXME: If we have both nuw and nsw, we should reduce the range further.
if (IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(&BO))) { if (IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(&BO))) {
// 'add nuw x, C' produces [C, UINT_MAX]. // 'add nuw x, C' produces [C, UINT_MAX].
Lower = *C; Lower = *C;
} else if (IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(&BO))) { } else if (IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(&BO))) {
if (C->isNegative()) { if (C->isNegative()) {
// 'add nsw x, -C' produces [SINT_MIN, SINT_MAX - C]. // 'add nsw x, -C' produces [SINT_MIN, SINT_MAX - C].
Lower = APInt::getSignedMinValue(Width); Lower = APInt::getSignedMinValue(Width);
Show All 21 Linesstatic void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower,
case Instruction::AShr: case Instruction::AShr:
if (match(BO.getOperand(1), m_APInt(C)) && C->ult(Width)) { if (match(BO.getOperand(1), m_APInt(C)) && C->ult(Width)) {
// 'ashr x, C' produces [INT_MIN >> C, INT_MAX >> C]. // 'ashr x, C' produces [INT_MIN >> C, INT_MAX >> C].
Lower = APInt::getSignedMinValue(Width).ashr(*C); Lower = APInt::getSignedMinValue(Width).ashr(*C);
Upper = APInt::getSignedMaxValue(Width).ashr(*C) + 1; Upper = APInt::getSignedMaxValue(Width).ashr(*C) + 1;
} else if (match(BO.getOperand(0), m_APInt(C))) { } else if (match(BO.getOperand(0), m_APInt(C))) {
unsigned ShiftAmount = Width - 1; unsigned ShiftAmount = Width - 1;
if (!C->isNullValue() && IIQ.isExact(&BO)) if (!C->isZero() && IIQ.isExact(&BO))
ShiftAmount = C->countTrailingZeros(); ShiftAmount = C->countTrailingZeros();
if (C->isNegative()) { if (C->isNegative()) {
// 'ashr C, x' produces [C, C >> (Width-1)] // 'ashr C, x' produces [C, C >> (Width-1)]
Lower = *C; Lower = *C;
Upper = C->ashr(ShiftAmount) + 1; Upper = C->ashr(ShiftAmount) + 1;
} else { } else {
// 'ashr C, x' produces [C >> (Width-1), C] // 'ashr C, x' produces [C >> (Width-1), C]
Lower = C->ashr(ShiftAmount); Lower = C->ashr(ShiftAmount);
Upper = *C + 1; Upper = *C + 1;
} }
} }
break; break;
case Instruction::LShr: case Instruction::LShr:
if (match(BO.getOperand(1), m_APInt(C)) && C->ult(Width)) { if (match(BO.getOperand(1), m_APInt(C)) && C->ult(Width)) {
// 'lshr x, C' produces [0, UINT_MAX >> C]. // 'lshr x, C' produces [0, UINT_MAX >> C].
Upper = APInt::getAllOnes(Width).lshr(*C) + 1; Upper = APInt::getAllOnes(Width).lshr(*C) + 1;
} else if (match(BO.getOperand(0), m_APInt(C))) { } else if (match(BO.getOperand(0), m_APInt(C))) {
// 'lshr C, x' produces [C >> (Width-1), C]. // 'lshr C, x' produces [C >> (Width-1), C].
unsigned ShiftAmount = Width - 1; unsigned ShiftAmount = Width - 1;
if (!C->isNullValue() && IIQ.isExact(&BO)) if (!C->isZero() && IIQ.isExact(&BO))
ShiftAmount = C->countTrailingZeros(); ShiftAmount = C->countTrailingZeros();
Lower = C->lshr(ShiftAmount); Lower = C->lshr(ShiftAmount);
Upper = *C + 1; Upper = *C + 1;
} }
break; break;
case Instruction::Shl: case Instruction::Shl:
if (match(BO.getOperand(0), m_APInt(C))) { if (match(BO.getOperand(0), m_APInt(C))) {
Show All 16 Lines if (match(BO.getOperand(0), m_APInt(C))) {
} }
} }
break; break;
case Instruction::SDiv: case Instruction::SDiv:
if (match(BO.getOperand(1), m_APInt(C))) { if (match(BO.getOperand(1), m_APInt(C))) {
APInt IntMin = APInt::getSignedMinValue(Width); APInt IntMin = APInt::getSignedMinValue(Width);
APInt IntMax = APInt::getSignedMaxValue(Width); APInt IntMax = APInt::getSignedMaxValue(Width);
if (C->isAllOnesValue()) { if (C->isAllOnes()) {
// 'sdiv x, -1' produces [INT_MIN + 1, INT_MAX] // 'sdiv x, -1' produces [INT_MIN + 1, INT_MAX]
// where C != -1 and C != 0 and C != 1 // where C != -1 and C != 0 and C != 1
Lower = IntMin + 1; Lower = IntMin + 1;
Upper = IntMax + 1; Upper = IntMax + 1;
} else if (C->countLeadingZeros() < Width - 1) { } else if (C->countLeadingZeros() < Width - 1) {
// 'sdiv x, C' produces [INT_MIN / C, INT_MAX / C] // 'sdiv x, C' produces [INT_MIN / C, INT_MAX / C]
// where C != -1 and C != 0 and C != 1 // where C != -1 and C != 0 and C != 1
Lower = IntMin.sdiv(*C); Lower = IntMin.sdiv(*C);
Show All 12 Lines if (match(BO.getOperand(1), m_APInt(C))) {
// 'sdiv C, x' produces [-|C|, |C|]. // 'sdiv C, x' produces [-|C|, |C|].
Upper = C->abs() + 1; Upper = C->abs() + 1;
Lower = (-Upper) + 1; Lower = (-Upper) + 1;
} }
} }
break; break;
case Instruction::UDiv: case Instruction::UDiv:
if (match(BO.getOperand(1), m_APInt(C)) && !C->isNullValue()) { if (match(BO.getOperand(1), m_APInt(C)) && !C->isZero()) {
// 'udiv x, C' produces [0, UINT_MAX / C]. // 'udiv x, C' produces [0, UINT_MAX / C].
Upper = APInt::getMaxValue(Width).udiv(*C) + 1; Upper = APInt::getMaxValue(Width).udiv(*C) + 1;
} else if (match(BO.getOperand(0), m_APInt(C))) { } else if (match(BO.getOperand(0), m_APInt(C))) {
// 'udiv C, x' produces [0, C]. // 'udiv C, x' produces [0, C].
Upper = *C + 1; Upper = *C + 1;
} }
break; break;
▲ Show 20 Lines • Show All 343 Lines • Show Last 20 Lines

llvm/lib/CodeGen/CodeGenPrepare.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 4,181 Lines • ▼ Show 20 Linesbool TypePromotionHelper::canGetThrough(const Instruction *Inst,
if ((Inst->getOpcode() == Instruction::And || if ((Inst->getOpcode() == Instruction::And ||
Inst->getOpcode() == Instruction::Or)) Inst->getOpcode() == Instruction::Or))
return true; return true;
// ext(xor(opnd, cst)) --> xor(ext(opnd), ext(cst)) // ext(xor(opnd, cst)) --> xor(ext(opnd), ext(cst))
if (Inst->getOpcode() == Instruction::Xor) { if (Inst->getOpcode() == Instruction::Xor) {
const ConstantInt *Cst = dyn_cast<ConstantInt>(Inst->getOperand(1)); const ConstantInt *Cst = dyn_cast<ConstantInt>(Inst->getOperand(1));
// Make sure it is not a NOT. // Make sure it is not a NOT.
if (Cst && !Cst->getValue().isAllOnesValue()) if (Cst && !Cst->getValue().isAllOnes())
return true; return true;
} }
// zext(shrl(opnd, cst)) --> shrl(zext(opnd), zext(cst)) // zext(shrl(opnd, cst)) --> shrl(zext(opnd), zext(cst))
// It may change a poisoned value into a regular value, like // It may change a poisoned value into a regular value, like
// zext i32 (shrl i8 %val, 12) --> shrl i32 (zext i8 %val), 12 // zext i32 (shrl i8 %val, 12) --> shrl i32 (zext i8 %val), 12
// poisoned value regular value // poisoned value regular value
// It should be OK since undef covers valid value. // It should be OK since undef covers valid value.
▲ Show 20 Lines • Show All 4,109 Lines • Show Last 20 Lines

llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

Show First 20 Lines • Show All 2,762 Lines • ▼ Show 20 Linesbool CombinerHelper::matchRedundantOr(MachineInstr &MI, Register &Replacement) {
// Check that x | Mask == x. // Check that x | Mask == x.
// x | 0 == x, always // x | 0 == x, always
// x | 1 == x, only if x is also 1 // x | 1 == x, only if x is also 1
// Meaning Mask has no effect if every bit is either zero in Mask or one in x. // Meaning Mask has no effect if every bit is either zero in Mask or one in x.
// //
// Check if we can replace OrDst with the LHS of the G_OR // Check if we can replace OrDst with the LHS of the G_OR
if (canReplaceReg(OrDst, LHS, MRI) && if (canReplaceReg(OrDst, LHS, MRI) &&
(LHSBits.One | RHSBits.Zero).isAllOnesValue()) { (LHSBits.One | RHSBits.Zero).isAllOnes()) {
Replacement = LHS; Replacement = LHS;
return true; return true;
} }
// Check if we can replace OrDst with the RHS of the G_OR // Check if we can replace OrDst with the RHS of the G_OR
if (canReplaceReg(OrDst, RHS, MRI) && if (canReplaceReg(OrDst, RHS, MRI) &&
(LHSBits.Zero | RHSBits.One).isAllOnesValue()) { (LHSBits.Zero | RHSBits.One).isAllOnes()) {
Replacement = RHS; Replacement = RHS;
return true; return true;
} }
return false; return false;
} }
bool CombinerHelper::matchRedundantSExtInReg(MachineInstr &MI) { bool CombinerHelper::matchRedundantSExtInReg(MachineInstr &MI) {
▲ Show 20 Lines • Show All 1,630 Lines • Show Last 20 Lines

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 4,814 Lines • ▼ Show 20 Lines
LegalizerHelper::LegalizeResult LegalizerHelper::LegalizeResult
LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
const LLT HalfTy, const LLT AmtTy) { const LLT HalfTy, const LLT AmtTy) {
Register InL = MRI.createGenericVirtualRegister(HalfTy); Register InL = MRI.createGenericVirtualRegister(HalfTy);
Register InH = MRI.createGenericVirtualRegister(HalfTy); Register InH = MRI.createGenericVirtualRegister(HalfTy);
MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1)); MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1));
if (Amt.isNullValue()) { if (Amt.isZero()) {
MIRBuilder.buildMerge(MI.getOperand(0), {InL, InH}); MIRBuilder.buildMerge(MI.getOperand(0), {InL, InH});
MI.eraseFromParent(); MI.eraseFromParent();
return Legalized; return Legalized;
} }
LLT NVT = HalfTy; LLT NVT = HalfTy;
unsigned NVTBits = HalfTy.getSizeInBits(); unsigned NVTBits = HalfTy.getSizeInBits();
unsigned VTBits = 2 * NVTBits; unsigned VTBits = 2 * NVTBits;
▲ Show 20 Lines • Show All 3,158 Lines • Show Last 20 Lines

llvm/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp

Show First 20 Lines • Show All 564 Lines • ▼ Show 20 Lines for (const RegisterBankInfo::PartialMapping &PartMap : *this) {
// without overlaps. // without overlaps.
// The high bit is exclusive in the APInt API, thus getHighBitIdx + 1. // The high bit is exclusive in the APInt API, thus getHighBitIdx + 1.
APInt PartMapMask = APInt::getBitsSet(OrigValueBitWidth, PartMap.StartIdx, APInt PartMapMask = APInt::getBitsSet(OrigValueBitWidth, PartMap.StartIdx,
PartMap.getHighBitIdx() + 1); PartMap.getHighBitIdx() + 1);
ValueMask ^= PartMapMask; ValueMask ^= PartMapMask;
assert((ValueMask & PartMapMask) == PartMapMask && assert((ValueMask & PartMapMask) == PartMapMask &&
"Some partial mappings overlap"); "Some partial mappings overlap");
} }
assert(ValueMask.isAllOnesValue() && "Value is not fully mapped"); assert(ValueMask.isAllOnes() && "Value is not fully mapped");
return true; return true;
} }
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD void RegisterBankInfo::ValueMapping::dump() const { LLVM_DUMP_METHOD void RegisterBankInfo::ValueMapping::dump() const {
print(dbgs()); print(dbgs());
dbgs() << '\n'; dbgs() << '\n';
} }
▲ Show 20 Lines • Show All 224 Lines • Show Last 20 Lines

llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp

Show First 20 Lines • Show All 302 Lines • ▼ Show 20 Lines Polynomial &mul(const APInt &C) {
// [qed] // [qed]
if (C.getBitWidth() != A.getBitWidth()) { if (C.getBitWidth() != A.getBitWidth()) {
ErrorMSBs = (unsigned)-1; ErrorMSBs = (unsigned)-1;
return *this; return *this;
} }
// Multiplying by one is a no-op. // Multiplying by one is a no-op.
if (C.isOneValue()) { if (C.isOne()) {
return *this; return *this;
} }
// Multiplying by zero removes the coefficient B and defines all bits. // Multiplying by zero removes the coefficient B and defines all bits.
if (C.isZero()) { if (C.isZero()) {
ErrorMSBs = 0; ErrorMSBs = 0;
deleteB(); deleteB();
} }
▲ Show 20 Lines • Show All 1,047 Lines • Show Last 20 Lines

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 9,415 Lines • ▼ Show 20 Lines if (TLI.convertSelectOfConstantsToMath(VT)) {
if (C1Val + 1 == C2Val) { if (C1Val + 1 == C2Val) {
// select Cond, C1, C1+1 --> add (sext Cond), C1+1 // select Cond, C1, C1+1 --> add (sext Cond), C1+1
if (VT != MVT::i1) if (VT != MVT::i1)
Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond); Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
} }
// select Cond, Pow2, 0 --> (zext Cond) << log2(Pow2) // select Cond, Pow2, 0 --> (zext Cond) << log2(Pow2)
if (C1Val.isPowerOf2() && C2Val.isNullValue()) { if (C1Val.isPowerOf2() && C2Val.isZero()) {
if (VT != MVT::i1) if (VT != MVT::i1)
Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
SDValue ShAmtC = DAG.getConstant(C1Val.exactLogBase2(), DL, VT); SDValue ShAmtC = DAG.getConstant(C1Val.exactLogBase2(), DL, VT);
return DAG.getNode(ISD::SHL, DL, VT, Cond, ShAmtC); return DAG.getNode(ISD::SHL, DL, VT, Cond, ShAmtC);
} }
if (SDValue V = foldSelectOfConstantsUsingSra(N, DAG)) if (SDValue V = foldSelectOfConstantsUsingSra(N, DAG))
return V; return V;
▲ Show 20 Lines • Show All 1,834 Lines • ▼ Show 20 Lines if (isNullOrNullSplat(Op0))
Op = Op1; Op = Op1;
else if (isNullOrNullSplat(Op1)) else if (isNullOrNullSplat(Op1))
Op = Op0; Op = Op0;
else else
return false; return false;
Known = DAG.computeKnownBits(Op); Known = DAG.computeKnownBits(Op);
return (Known.Zero | 1).isAllOnesValue(); return (Known.Zero | 1).isAllOnes();
} }
/// Given an extending node with a pop-count operand, if the target does not /// Given an extending node with a pop-count operand, if the target does not
/// support a pop-count in the narrow source type but does support it in the /// support a pop-count in the narrow source type but does support it in the
/// destination type, widen the pop-count to the destination type. /// destination type, widen the pop-count to the destination type.
static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG) { static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG) {
assert((Extend->getOpcode() == ISD::ZERO_EXTEND || assert((Extend->getOpcode() == ISD::ZERO_EXTEND ||
Extend->getOpcode() == ISD::ANY_EXTEND) && "Expected extend op"); Extend->getOpcode() == ISD::ANY_EXTEND) && "Expected extend op");
▲ Show 20 Lines • Show All 5,020 Lines • ▼ Show 20 Lines
}; };
} // end anonymous namespace } // end anonymous namespace
/// Check that all bits set in \p UsedBits form a dense region, i.e., /// Check that all bits set in \p UsedBits form a dense region, i.e.,
/// \p UsedBits looks like 0..0 1..1 0..0. /// \p UsedBits looks like 0..0 1..1 0..0.
static bool areUsedBitsDense(const APInt &UsedBits) { static bool areUsedBitsDense(const APInt &UsedBits) {
// If all the bits are one, this is dense! // If all the bits are one, this is dense!
if (UsedBits.isAllOnesValue()) if (UsedBits.isAllOnes())
return true; return true;
// Get rid of the unused bits on the right. // Get rid of the unused bits on the right.
APInt NarrowedUsedBits = UsedBits.lshr(UsedBits.countTrailingZeros()); APInt NarrowedUsedBits = UsedBits.lshr(UsedBits.countTrailingZeros());
// Get rid of the unused bits on the left. // Get rid of the unused bits on the left.
if (NarrowedUsedBits.countLeadingZeros()) if (NarrowedUsedBits.countLeadingZeros())
NarrowedUsedBits = NarrowedUsedBits.trunc(NarrowedUsedBits.getActiveBits()); NarrowedUsedBits = NarrowedUsedBits.trunc(NarrowedUsedBits.getActiveBits());
// Check that the chunk of bits is completely used. // Check that the chunk of bits is completely used.
return NarrowedUsedBits.isAllOnesValue(); return NarrowedUsedBits.isAllOnes();
} }
/// Check whether or not \p First and \p Second are next to each other /// Check whether or not \p First and \p Second are next to each other
/// in memory. This means that there is no hole between the bits loaded /// in memory. This means that there is no hole between the bits loaded
/// by \p First and the bits loaded by \p Second. /// by \p First and the bits loaded by \p Second.
static bool areSlicesNextToEachOther(const LoadedSlice &First, static bool areSlicesNextToEachOther(const LoadedSlice &First,
const LoadedSlice &Second) { const LoadedSlice &Second) {
assert(First.Origin == Second.Origin && First.Origin && assert(First.Origin == Second.Origin && First.Origin &&
▲ Show 20 Lines • Show All 402 Lines • ▼ Show 20 Lines if (LD->getBasePtr() != Ptr ||
return SDValue(); return SDValue();
// Find the type to narrow it the load / op / store to. // Find the type to narrow it the load / op / store to.
SDValue N1 = Value.getOperand(1); SDValue N1 = Value.getOperand(1);
unsigned BitWidth = N1.getValueSizeInBits(); unsigned BitWidth = N1.getValueSizeInBits();
APInt Imm = cast<ConstantSDNode>(N1)->getAPIntValue(); APInt Imm = cast<ConstantSDNode>(N1)->getAPIntValue();
if (Opc == ISD::AND) if (Opc == ISD::AND)
Imm ^= APInt::getAllOnes(BitWidth); Imm ^= APInt::getAllOnes(BitWidth);
if (Imm == 0 || Imm.isAllOnesValue()) if (Imm == 0 || Imm.isAllOnes())
return SDValue(); return SDValue();
unsigned ShAmt = Imm.countTrailingZeros(); unsigned ShAmt = Imm.countTrailingZeros();
unsigned MSB = BitWidth - Imm.countLeadingZeros() - 1; unsigned MSB = BitWidth - Imm.countLeadingZeros() - 1;
unsigned NewBW = NextPowerOf2(MSB - ShAmt); unsigned NewBW = NextPowerOf2(MSB - ShAmt);
EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW); EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW);
// The narrowing should be profitable, the load/store operation should be // The narrowing should be profitable, the load/store operation should be
// legal (or custom) and the store size should be equal to the NewVT width. // legal (or custom) and the store size should be equal to the NewVT width.
while (NewBW < BitWidth && while (NewBW < BitWidth &&
▲ Show 20 Lines • Show All 5,381 Lines • ▼ Show 20 Lines for (int i = 0; i != NumSubElts; ++i) {
return SDValue(); return SDValue();
// Extract the sub element from the constant bit mask. // Extract the sub element from the constant bit mask.
if (DAG.getDataLayout().isBigEndian()) if (DAG.getDataLayout().isBigEndian())
Bits = Bits.extractBits(NumSubBits, (Split - SubIdx - 1) * NumSubBits); Bits = Bits.extractBits(NumSubBits, (Split - SubIdx - 1) * NumSubBits);
else else
Bits = Bits.extractBits(NumSubBits, SubIdx * NumSubBits); Bits = Bits.extractBits(NumSubBits, SubIdx * NumSubBits);
if (Bits.isAllOnesValue()) if (Bits.isAllOnes())
Indices.push_back(i); Indices.push_back(i);
else if (Bits == 0) else if (Bits == 0)
Indices.push_back(i + NumSubElts); Indices.push_back(i + NumSubElts);
else else
return SDValue(); return SDValue();
} }
// Let's see if the target supports this vector_shuffle. // Let's see if the target supports this vector_shuffle.
▲ Show 20 Lines • Show All 1,529 Lines • Show Last 20 Lines

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 169 Lines • ▼ Show 20 Lines
bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) { bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
// Look through a bit convert. // Look through a bit convert.
while (N->getOpcode() == ISD::BITCAST) while (N->getOpcode() == ISD::BITCAST)
N = N->getOperand(0).getNode(); N = N->getOperand(0).getNode();
if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
APInt SplatVal; APInt SplatVal;
return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnesValue(); return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
} }
if (N->getOpcode() != ISD::BUILD_VECTOR) return false; if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
unsigned i = 0, e = N->getNumOperands(); unsigned i = 0, e = N->getNumOperands();
// Skip over all of the undef values. // Skip over all of the undef values.
while (i != e && N->getOperand(i).isUndef()) while (i != e && N->getOperand(i).isUndef())
Show All 32 Lines
bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) { bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
// Look through a bit convert. // Look through a bit convert.
while (N->getOpcode() == ISD::BITCAST) while (N->getOpcode() == ISD::BITCAST)
N = N->getOperand(0).getNode(); N = N->getOperand(0).getNode();
if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
APInt SplatVal; APInt SplatVal;
return isConstantSplatVector(N, SplatVal) && SplatVal.isNullValue(); return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
} }
if (N->getOpcode() != ISD::BUILD_VECTOR) return false; if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
bool IsAllUndef = true; bool IsAllUndef = true;
for (const SDValue &Op : N->op_values()) { for (const SDValue &Op : N->op_values()) {
if (Op.isUndef()) if (Op.isUndef())
continue; continue;
▲ Show 20 Lines • Show All 10,857 Lines • Show Last 20 Lines

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 667 Lines • ▼ Show 20 Lines if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&
DAG.getDataLayout().isLittleEndian()) { DAG.getDataLayout().isLittleEndian()) {
unsigned Scale = NumDstEltBits / NumSrcEltBits; unsigned Scale = NumDstEltBits / NumSrcEltBits;
unsigned NumSrcElts = SrcVT.getVectorNumElements(); unsigned NumSrcElts = SrcVT.getVectorNumElements();
APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits); APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
APInt DemandedSrcElts = APInt::getZero(NumSrcElts); APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
for (unsigned i = 0; i != Scale; ++i) { for (unsigned i = 0; i != Scale; ++i) {
unsigned Offset = i * NumSrcEltBits; unsigned Offset = i * NumSrcEltBits;
APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset); APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
if (!Sub.isNullValue()) { if (!Sub.isZero()) {
DemandedSrcBits |= Sub; DemandedSrcBits |= Sub;
for (unsigned j = 0; j != NumElts; ++j) for (unsigned j = 0; j != NumElts; ++j)
if (DemandedElts[j]) if (DemandedElts[j])
DemandedSrcElts.setBit((j * Scale) + i); DemandedSrcElts.setBit((j * Scale) + i);
} }
} }
if (SDValue V = SimplifyMultipleUseDemandedBits( if (SDValue V = SimplifyMultipleUseDemandedBits(
▲ Show 20 Lines • Show All 923 Lines • ▼ Show 20 Lines case ISD::SRA: {
if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >= if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
NumHiDemandedBits) NumHiDemandedBits)
return TLO.CombineTo(Op, Op0); return TLO.CombineTo(Op, Op0);
// If this is an arithmetic shift right and only the low-bit is set, we can // If this is an arithmetic shift right and only the low-bit is set, we can
// always convert this into a logical shr, even if the shift amount is // always convert this into a logical shr, even if the shift amount is
// variable. The low bit of the shift cannot be an input sign bit unless // variable. The low bit of the shift cannot be an input sign bit unless
// the shift amount is >= the size of the datatype, which is undefined. // the shift amount is >= the size of the datatype, which is undefined.
if (DemandedBits.isOneValue()) if (DemandedBits.isOne())
return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
if (const APInt *SA = if (const APInt *SA =
TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) { TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
unsigned ShAmt = SA->getZExtValue(); unsigned ShAmt = SA->getZExtValue();
if (ShAmt == 0) if (ShAmt == 0)
return TLO.CombineTo(Op, Op0); return TLO.CombineTo(Op, Op0);
▲ Show 20 Lines • Show All 159 Lines • ▼ Show 20 Lines case ISD::BSWAP: {
Known.One = Known2.One.byteSwap(); Known.One = Known2.One.byteSwap();
Known.Zero = Known2.Zero.byteSwap(); Known.Zero = Known2.Zero.byteSwap();
break; break;
} }
case ISD::CTPOP: { case ISD::CTPOP: {
// If only 1 bit is demanded, replace with PARITY as long as we're before // If only 1 bit is demanded, replace with PARITY as long as we're before
// op legalization. // op legalization.
// FIXME: Limit to scalars for now. // FIXME: Limit to scalars for now.
if (DemandedBits.isOneValue() && !TLO.LegalOps && !VT.isVector()) if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT, return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
Op.getOperand(0))); Op.getOperand(0)));
Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth); Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
break; break;
} }
case ISD::SIGN_EXTEND_INREG: { case ISD::SIGN_EXTEND_INREG: {
SDValue Op0 = Op.getOperand(0); SDValue Op0 = Op.getOperand(0);
▲ Show 20 Lines • Show All 344 Lines • ▼ Show 20 Lines if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&
TLO.DAG.getDataLayout().isLittleEndian()) { TLO.DAG.getDataLayout().isLittleEndian()) {
unsigned Scale = BitWidth / NumSrcEltBits; unsigned Scale = BitWidth / NumSrcEltBits;
unsigned NumSrcElts = SrcVT.getVectorNumElements(); unsigned NumSrcElts = SrcVT.getVectorNumElements();
APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits); APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
APInt DemandedSrcElts = APInt::getZero(NumSrcElts); APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
for (unsigned i = 0; i != Scale; ++i) { for (unsigned i = 0; i != Scale; ++i) {
unsigned Offset = i * NumSrcEltBits; unsigned Offset = i * NumSrcEltBits;
APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset); APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
if (!Sub.isNullValue()) { if (!Sub.isZero()) {
DemandedSrcBits |= Sub; DemandedSrcBits |= Sub;
for (unsigned j = 0; j != NumElts; ++j) for (unsigned j = 0; j != NumElts; ++j)
if (DemandedElts[j]) if (DemandedElts[j])
DemandedSrcElts.setBit((j * Scale) + i); DemandedSrcElts.setBit((j * Scale) + i);
} }
} }
APInt KnownSrcUndef, KnownSrcZero; APInt KnownSrcUndef, KnownSrcZero;
▲ Show 20 Lines • Show All 943 Lines • ▼ Show 20 Linesbool TargetLowering::isConstTrueVal(const SDNode *N) const {
} else { } else {
return false; return false;
} }
switch (getBooleanContents(N->getValueType(0))) { switch (getBooleanContents(N->getValueType(0))) {
case UndefinedBooleanContent: case UndefinedBooleanContent:
return CVal[0]; return CVal[0];
case ZeroOrOneBooleanContent: case ZeroOrOneBooleanContent:
return CVal.isOneValue(); return CVal.isOne();
case ZeroOrNegativeOneBooleanContent: case ZeroOrNegativeOneBooleanContent:
return CVal.isAllOnes(); return CVal.isAllOnes();
} }
llvm_unreachable("Invalid boolean contents"); llvm_unreachable("Invalid boolean contents");
} }
bool TargetLowering::isConstFalseVal(const SDNode *N) const { bool TargetLowering::isConstFalseVal(const SDNode *N) const {
▲ Show 20 Lines • Show All 197 Lines • ▼ Show 20 LinesSDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
return T2; return T2;
} }
// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift( SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond, EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
DAGCombinerInfo &DCI, const SDLoc &DL) const { DAGCombinerInfo &DCI, const SDLoc &DL) const {
assert(isConstOrConstSplat(N1C) && assert(isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
"Should be a comparison with 0."); "Should be a comparison with 0.");
assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
"Valid only for [in]equality comparisons."); "Valid only for [in]equality comparisons.");
unsigned NewShiftOpcode; unsigned NewShiftOpcode;
SDValue X, C, Y; SDValue X, C, Y;
SelectionDAG &DAG = DCI.DAG; SelectionDAG &DAG = DCI.DAG;
▲ Show 20 Lines • Show All 206 Lines • ▼ Show 20 Lines if (auto *N1C = isConstOrConstSplat(N1)) {
// Optimize some CTPOP cases. // Optimize some CTPOP cases.
if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG)) if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
return V; return V;
// If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
// equality comparison, then we're just comparing whether X itself is // equality comparison, then we're just comparing whether X itself is
// zero. // zero.
if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) && if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
N0.getOperand(0).getOpcode() == ISD::CTLZ && N0.getOperand(0).getOpcode() == ISD::CTLZ &&
isPowerOf2_32(N0.getScalarValueSizeInBits())) { isPowerOf2_32(N0.getScalarValueSizeInBits())) {
if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) { if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) { ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
if ((C1 == 0) == (Cond == ISD::SETEQ)) { if ((C1 == 0) == (Cond == ISD::SETEQ)) {
// (srl (ctlz x), 5) == 0 -> X != 0 // (srl (ctlz x), 5) == 0 -> X != 0
// (srl (ctlz x), 5) != 1 -> X != 0 // (srl (ctlz x), 5) != 1 -> X != 0
▲ Show 20 Lines • Show All 456 Lines • ▼ Show 20 Lines if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift( if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
VT, N0, N1, Cond, DCI, dl)) VT, N0, N1, Cond, DCI, dl))
return CC; return CC;
// For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y). // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
// For example, when high 32-bits of i64 X are known clear: // For example, when high 32-bits of i64 X are known clear:
// all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0 // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
// all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1 // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
bool CmpZero = N1C->getAPIntValue().isNullValue(); bool CmpZero = N1C->getAPIntValue().isZero();
bool CmpNegOne = N1C->getAPIntValue().isAllOnes(); bool CmpNegOne = N1C->getAPIntValue().isAllOnes();
if ((CmpZero || CmpNegOne) && N0.hasOneUse()) { if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
// Match or(lo,shl(hi,bw/2)) pattern. // Match or(lo,shl(hi,bw/2)) pattern.
auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) { auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
unsigned EltBits = V.getScalarValueSizeInBits(); unsigned EltBits = V.getScalarValueSizeInBits();
if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0) if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
return false; return false;
SDValue LHS = V.getOperand(0); SDValue LHS = V.getOperand(0);
▲ Show 20 Lines • Show All 1,133 Lines • ▼ Show 20 Lines auto BuildSDIVPattern = [&](ConstantSDNode *C) {
if (C->isZero()) if (C->isZero())
return false; return false;
const APInt &Divisor = C->getAPIntValue(); const APInt &Divisor = C->getAPIntValue();
SignedDivisionByConstantInfo magics = SignedDivisionByConstantInfo::get(Divisor); SignedDivisionByConstantInfo magics = SignedDivisionByConstantInfo::get(Divisor);
int NumeratorFactor = 0; int NumeratorFactor = 0;
int ShiftMask = -1; int ShiftMask = -1;
if (Divisor.isOneValue() || Divisor.isAllOnes()) { if (Divisor.isOne() || Divisor.isAllOnes()) {
// If d is +1/-1, we just multiply the numerator by +1/-1. // If d is +1/-1, we just multiply the numerator by +1/-1.
NumeratorFactor = Divisor.getSExtValue(); NumeratorFactor = Divisor.getSExtValue();
magics.Magic = 0; magics.Magic = 0;
magics.ShiftAmount = 0; magics.ShiftAmount = 0;
ShiftMask = 0; ShiftMask = 0;
} else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) { } else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) {
// If d > 0 and m < 0, add the numerator. // If d > 0 and m < 0, add the numerator.
NumeratorFactor = 1; NumeratorFactor = 1;
▲ Show 20 Lines • Show All 140 Lines • ▼ Show 20 Lines if (magics.IsAdd != 0 && !Divisor[0]) {
// Get magic number for the shifted divisor. // Get magic number for the shifted divisor.
magics = UnsignedDivisonByConstantInfo::get(Divisor.lshr(PreShift), PreShift); magics = UnsignedDivisonByConstantInfo::get(Divisor.lshr(PreShift), PreShift);
assert(magics.IsAdd == 0 && "Should use cheap fixup now"); assert(magics.IsAdd == 0 && "Should use cheap fixup now");
} }
APInt Magic = magics.Magic; APInt Magic = magics.Magic;
unsigned SelNPQ; unsigned SelNPQ;
if (magics.IsAdd == 0 || Divisor.isOneValue()) { if (magics.IsAdd == 0 || Divisor.isOne()) {
assert(magics.ShiftAmount < Divisor.getBitWidth() && assert(magics.ShiftAmount < Divisor.getBitWidth() &&
"We shouldn't generate an undefined shift!"); "We shouldn't generate an undefined shift!");
PostShift = magics.ShiftAmount; PostShift = magics.ShiftAmount;
SelNPQ = false; SelNPQ = false;
} else { } else {
PostShift = magics.ShiftAmount - 1; PostShift = magics.ShiftAmount - 1;
SelNPQ = true; SelNPQ = true;
} }
▲ Show 20 Lines • Show All 183 Lines • ▼ Show 20 LinesTargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) { auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
// Division by 0 is UB. Leave it to be constant-folded elsewhere. // Division by 0 is UB. Leave it to be constant-folded elsewhere.
if (CDiv->isZero()) if (CDiv->isZero())
return false; return false;
const APInt &D = CDiv->getAPIntValue(); const APInt &D = CDiv->getAPIntValue();
const APInt &Cmp = CCmp->getAPIntValue(); const APInt &Cmp = CCmp->getAPIntValue();
ComparingWithAllZeros &= Cmp.isNullValue(); ComparingWithAllZeros &= Cmp.isZero();
// x u% C1` is *always* less than C1. So given `x u% C1 == C2`, // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
// if C2 is not less than C1, the comparison is always false. // if C2 is not less than C1, the comparison is always false.
// But we will only be able to produce the comparison that will give the // But we will only be able to produce the comparison that will give the
// opposive tautological answer. So this lane would need to be fixed up. // opposive tautological answer. So this lane would need to be fixed up.
bool TautologicalInvertedLane = D.ule(Cmp); bool TautologicalInvertedLane = D.ule(Cmp);
HadTautologicalInvertedLanes |= TautologicalInvertedLane; HadTautologicalInvertedLanes |= TautologicalInvertedLane;
// If all lanes are tautological (either all divisors are ones, or divisor // If all lanes are tautological (either all divisors are ones, or divisor
// is not greater than the constant we are comparing with), // is not greater than the constant we are comparing with),
// we will prefer to avoid the fold. // we will prefer to avoid the fold.
bool TautologicalLane = D.isOneValue() || TautologicalInvertedLane; bool TautologicalLane = D.isOne() || TautologicalInvertedLane;
HadTautologicalLanes |= TautologicalLane; HadTautologicalLanes |= TautologicalLane;
AllLanesAreTautological &= TautologicalLane; AllLanesAreTautological &= TautologicalLane;
// If we are comparing with non-zero, we need'll need to subtract said // If we are comparing with non-zero, we need'll need to subtract said
// comparison value from the LHS. But there is no point in doing that if // comparison value from the LHS. But there is no point in doing that if
// every lane where we are comparing with non-zero is tautological.. // every lane where we are comparing with non-zero is tautological..
if (!Cmp.isNullValue()) if (!Cmp.isZero())
AllComparisonsWithNonZerosAreTautological &= TautologicalLane; AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
// Decompose D into D0 * 2^K // Decompose D into D0 * 2^K
unsigned K = D.countTrailingZeros(); unsigned K = D.countTrailingZeros();
assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate."); assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
APInt D0 = D.lshr(K); APInt D0 = D.lshr(K);
// D is even if it has trailing zeros. // D is even if it has trailing zeros.
HadEvenDivisor |= (K != 0); HadEvenDivisor |= (K != 0);
// D is a power-of-two if D0 is one. // D is a power-of-two if D0 is one.
// If all divisors are power-of-two, we will prefer to avoid the fold. // If all divisors are power-of-two, we will prefer to avoid the fold.
AllDivisorsArePowerOfTwo &= D0.isOneValue(); AllDivisorsArePowerOfTwo &= D0.isOne();
// P = inv(D0, 2^W) // P = inv(D0, 2^W)
// 2^W requires W + 1 bits, so we have to extend and then truncate. // 2^W requires W + 1 bits, so we have to extend and then truncate.
unsigned W = D.getBitWidth(); unsigned W = D.getBitWidth();
APInt P = D0.zext(W + 1) APInt P = D0.zext(W + 1)
.multiplicativeInverse(APInt::getSignedMinValue(W + 1)) .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
.trunc(W); .trunc(W);
assert(!P.isNullValue() && "No multiplicative inverse!"); // unreachable assert(!P.isZero() && "No multiplicative inverse!"); // unreachable
assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check."); assert((D0 * P).isOne() && "Multiplicative inverse sanity check.");
// Q = floor((2^W - 1) u/ D) // Q = floor((2^W - 1) u/ D)
// R = ((2^W - 1) u% D) // R = ((2^W - 1) u% D)
APInt Q, R; APInt Q, R;
APInt::udivrem(APInt::getAllOnes(W), D, Q, R); APInt::udivrem(APInt::getAllOnes(W), D, Q, R);
// If we are comparing with zero, then that comparison constant is okay, // If we are comparing with zero, then that comparison constant is okay,
// else it may need to be one less than that. // else it may need to be one less than that.
▲ Show 20 Lines • Show All 204 Lines • ▼ Show 20 Lines auto BuildSREMPattern = [&](ConstantSDNode *C) {
// WARNING: this fold is only valid for positive divisors! // WARNING: this fold is only valid for positive divisors!
APInt D = C->getAPIntValue(); APInt D = C->getAPIntValue();
if (D.isNegative()) if (D.isNegative())
D.negate(); // `rem %X, -C` is equivalent to `rem %X, C` D.negate(); // `rem %X, -C` is equivalent to `rem %X, C`
HadIntMinDivisor |= D.isMinSignedValue(); HadIntMinDivisor |= D.isMinSignedValue();
// If all divisors are ones, we will prefer to avoid the fold. // If all divisors are ones, we will prefer to avoid the fold.
HadOneDivisor |= D.isOneValue(); HadOneDivisor |= D.isOne();
AllDivisorsAreOnes &= D.isOneValue(); AllDivisorsAreOnes &= D.isOne();
// Decompose D into D0 * 2^K // Decompose D into D0 * 2^K
unsigned K = D.countTrailingZeros(); unsigned K = D.countTrailingZeros();
assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate."); assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
APInt D0 = D.lshr(K); APInt D0 = D.lshr(K);
if (!D.isMinSignedValue()) { if (!D.isMinSignedValue()) {
// D is even if it has trailing zeros; unless it's INT_MIN, in which case // D is even if it has trailing zeros; unless it's INT_MIN, in which case
// we don't care about this lane in this fold, we'll special-handle it. // we don't care about this lane in this fold, we'll special-handle it.
HadEvenDivisor |= (K != 0); HadEvenDivisor |= (K != 0);
} }
// D is a power-of-two if D0 is one. This includes INT_MIN. // D is a power-of-two if D0 is one. This includes INT_MIN.
// If all divisors are power-of-two, we will prefer to avoid the fold. // If all divisors are power-of-two, we will prefer to avoid the fold.
AllDivisorsArePowerOfTwo &= D0.isOneValue(); AllDivisorsArePowerOfTwo &= D0.isOne();
// P = inv(D0, 2^W) // P = inv(D0, 2^W)
// 2^W requires W + 1 bits, so we have to extend and then truncate. // 2^W requires W + 1 bits, so we have to extend and then truncate.
unsigned W = D.getBitWidth(); unsigned W = D.getBitWidth();
APInt P = D0.zext(W + 1) APInt P = D0.zext(W + 1)
.multiplicativeInverse(APInt::getSignedMinValue(W + 1)) .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
.trunc(W); .trunc(W);
assert(!P.isNullValue() && "No multiplicative inverse!"); // unreachable assert(!P.isZero() && "No multiplicative inverse!"); // unreachable
assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check."); assert((D0 * P).isOne() && "Multiplicative inverse sanity check.");
// A = floor((2^(W - 1) - 1) / D0) & -2^K // A = floor((2^(W - 1) - 1) / D0) & -2^K
APInt A = APInt::getSignedMaxValue(W).udiv(D0); APInt A = APInt::getSignedMaxValue(W).udiv(D0);
A.clearLowBits(K); A.clearLowBits(K);
if (!D.isMinSignedValue()) { if (!D.isMinSignedValue()) {
// If divisor INT_MIN, then we don't care about this lane in this fold, // If divisor INT_MIN, then we don't care about this lane in this fold,
// we'll special-handle it. // we'll special-handle it.
NeedToApplyOffset |= A != 0; NeedToApplyOffset |= A != 0;
} }
// Q = floor((2 * A) / (2^K)) // Q = floor((2 * A) / (2^K))
APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K)); APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));
assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) && assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) &&
"We are expecting that A is always less than all-ones for SVT"); "We are expecting that A is always less than all-ones for SVT");
assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
"We are expecting that K is always less than all-ones for ShSVT"); "We are expecting that K is always less than all-ones for ShSVT");
// If the divisor is 1 the result can be constant-folded. Likewise, we // If the divisor is 1 the result can be constant-folded. Likewise, we
// don't care about INT_MIN lanes, those can be set to undef if appropriate. // don't care about INT_MIN lanes, those can be set to undef if appropriate.
if (D.isOneValue()) { if (D.isOne()) {
// Set P, A and K to a bogus values so we can try to splat them. // Set P, A and K to a bogus values so we can try to splat them.
P = 0; P = 0;
A = -1; A = -1;
K = -1; K = -1;
// x ?% 1 == 0 <--> true <--> x u<= -1 // x ?% 1 == 0 <--> true <--> x u<= -1
Q = -1; Q = -1;
} }
▲ Show 20 Lines • Show All 3,123 Lines • Show Last 20 Lines

llvm/lib/IR/AsmWriter.cpp

Show First 20 Lines • Show All 1,760 Lines • ▼ Show 20 Linesvoid MDFieldPrinter::printInt(StringRef Name, IntTy Int, bool ShouldSkipZero) {
if (ShouldSkipZero && !Int) if (ShouldSkipZero && !Int)
return; return;
Out << FS << Name << ": " << Int; Out << FS << Name << ": " << Int;
} }
void MDFieldPrinter::printAPInt(StringRef Name, const APInt &Int, void MDFieldPrinter::printAPInt(StringRef Name, const APInt &Int,
bool IsUnsigned, bool ShouldSkipZero) { bool IsUnsigned, bool ShouldSkipZero) {
if (ShouldSkipZero && Int.isNullValue()) if (ShouldSkipZero && Int.isZero())
return; return;
Out << FS << Name << ": "; Out << FS << Name << ": ";
Int.print(Out, !IsUnsigned); Int.print(Out, !IsUnsigned);
} }
void MDFieldPrinter::printBool(StringRef Name, bool Value, void MDFieldPrinter::printBool(StringRef Name, bool Value,
Optional<bool> Default) { Optional<bool> Default) {
▲ Show 20 Lines • Show All 3,072 Lines • Show Last 20 Lines

llvm/lib/IR/ConstantFold.cpp

Show First 20 Lines • Show All 1,135 Lines • ▼ Show 20 Lines if (ConstantInt *CI2 = dyn_cast<ConstantInt>(C2)) {
return ConstantInt::get(CI1->getContext(), C1V - C2V); return ConstantInt::get(CI1->getContext(), C1V - C2V);
case Instruction::Mul: case Instruction::Mul:
return ConstantInt::get(CI1->getContext(), C1V * C2V); return ConstantInt::get(CI1->getContext(), C1V * C2V);
case Instruction::UDiv: case Instruction::UDiv:
assert(!CI2->isZero() && "Div by zero handled above"); assert(!CI2->isZero() && "Div by zero handled above");
return ConstantInt::get(CI1->getContext(), C1V.udiv(C2V)); return ConstantInt::get(CI1->getContext(), C1V.udiv(C2V));
case Instruction::SDiv: case Instruction::SDiv:
assert(!CI2->isZero() && "Div by zero handled above"); assert(!CI2->isZero() && "Div by zero handled above");
if (C2V.isAllOnesValue() && C1V.isMinSignedValue()) if (C2V.isAllOnes() && C1V.isMinSignedValue())
return PoisonValue::get(CI1->getType()); // MIN_INT / -1 -> poison return PoisonValue::get(CI1->getType()); // MIN_INT / -1 -> poison
return ConstantInt::get(CI1->getContext(), C1V.sdiv(C2V)); return ConstantInt::get(CI1->getContext(), C1V.sdiv(C2V));
case Instruction::URem: case Instruction::URem:
assert(!CI2->isZero() && "Div by zero handled above"); assert(!CI2->isZero() && "Div by zero handled above");
return ConstantInt::get(CI1->getContext(), C1V.urem(C2V)); return ConstantInt::get(CI1->getContext(), C1V.urem(C2V));
case Instruction::SRem: case Instruction::SRem:
assert(!CI2->isZero() && "Div by zero handled above"); assert(!CI2->isZero() && "Div by zero handled above");
if (C2V.isAllOnesValue() && C1V.isMinSignedValue()) if (C2V.isAllOnes() && C1V.isMinSignedValue())
return PoisonValue::get(CI1->getType()); // MIN_INT % -1 -> poison return PoisonValue::get(CI1->getType()); // MIN_INT % -1 -> poison
return ConstantInt::get(CI1->getContext(), C1V.srem(C2V)); return ConstantInt::get(CI1->getContext(), C1V.srem(C2V));
case Instruction::And: case Instruction::And:
return ConstantInt::get(CI1->getContext(), C1V & C2V); return ConstantInt::get(CI1->getContext(), C1V & C2V);
case Instruction::Or: case Instruction::Or:
return ConstantInt::get(CI1->getContext(), C1V | C2V); return ConstantInt::get(CI1->getContext(), C1V | C2V);
case Instruction::Xor: case Instruction::Xor:
return ConstantInt::get(CI1->getContext(), C1V ^ C2V); return ConstantInt::get(CI1->getContext(), C1V ^ C2V);
▲ Show 20 Lines • Show All 1,273 Lines • Show Last 20 Lines

llvm/lib/IR/ConstantRange.cpp

Show First 20 Lines • Show All 198 Lines • ▼ Show 20 Lines return ConstantRange::getNonEmpty(
APInt::Rounding::DOWN) + 1); APInt::Rounding::DOWN) + 1);
} }
/// Exact mul nsw region for single element RHS. /// Exact mul nsw region for single element RHS.
static ConstantRange makeExactMulNSWRegion(const APInt &V) { static ConstantRange makeExactMulNSWRegion(const APInt &V) {
// Handle special case for 0, -1 and 1. See the last for reason why we // Handle special case for 0, -1 and 1. See the last for reason why we
// specialize -1 and 1. // specialize -1 and 1.
unsigned BitWidth = V.getBitWidth(); unsigned BitWidth = V.getBitWidth();
if (V == 0 || V.isOneValue()) if (V == 0 || V.isOne())
return ConstantRange::getFull(BitWidth); return ConstantRange::getFull(BitWidth);
APInt MinValue = APInt::getSignedMinValue(BitWidth); APInt MinValue = APInt::getSignedMinValue(BitWidth);
APInt MaxValue = APInt::getSignedMaxValue(BitWidth); APInt MaxValue = APInt::getSignedMaxValue(BitWidth);
// e.g. Returning [-127, 127], represented as [-127, -128). // e.g. Returning [-127, 127], represented as [-127, -128).
if (V.isAllOnesValue()) if (V.isAllOnes())
return ConstantRange(-MaxValue, MinValue); return ConstantRange(-MaxValue, MinValue);
APInt Lower, Upper; APInt Lower, Upper;
if (V.isNegative()) { if (V.isNegative()) {
Lower = APIntOps::RoundingSDiv(MaxValue, V, APInt::Rounding::UP); Lower = APIntOps::RoundingSDiv(MaxValue, V, APInt::Rounding::UP);
Upper = APIntOps::RoundingSDiv(MinValue, V, APInt::Rounding::DOWN); Upper = APIntOps::RoundingSDiv(MinValue, V, APInt::Rounding::DOWN);
} else { } else {
Lower = APIntOps::RoundingSDiv(MinValue, V, APInt::Rounding::UP); Lower = APIntOps::RoundingSDiv(MinValue, V, APInt::Rounding::UP);
▲ Show 20 Lines • Show All 934 Lines • ▼ Show 20 Lines if (!NegL.isEmptySet() && !NegR.isEmptySet()) {
// We need to deal with one tricky case here: SignedMin / -1 is UB on the // We need to deal with one tricky case here: SignedMin / -1 is UB on the
// IR level, so we'll want to exclude this case when calculating bounds. // IR level, so we'll want to exclude this case when calculating bounds.
// (For APInts the operation is well-defined and yields SignedMin.) We // (For APInts the operation is well-defined and yields SignedMin.) We
// handle this by dropping either SignedMin from the LHS or -1 from the RHS. // handle this by dropping either SignedMin from the LHS or -1 from the RHS.
APInt Lo = (NegL.Upper - 1).sdiv(NegR.Lower); APInt Lo = (NegL.Upper - 1).sdiv(NegR.Lower);
if (NegL.Lower.isMinSignedValue() && NegR.Upper.isZero()) { if (NegL.Lower.isMinSignedValue() && NegR.Upper.isZero()) {
// Remove -1 from the LHS. Skip if it's the only element, as this would // Remove -1 from the LHS. Skip if it's the only element, as this would
// leave us with an empty set. // leave us with an empty set.
if (!NegR.Lower.isAllOnesValue()) { if (!NegR.Lower.isAllOnes()) {
APInt AdjNegRUpper; APInt AdjNegRUpper;
if (RHS.Lower.isAllOnesValue()) if (RHS.Lower.isAllOnes())
// Negative part of [-1, X] without -1 is [SignedMin, X]. // Negative part of [-1, X] without -1 is [SignedMin, X].
AdjNegRUpper = RHS.Upper; AdjNegRUpper = RHS.Upper;
else else
// [X, -1] without -1 is [X, -2]. // [X, -1] without -1 is [X, -2].
AdjNegRUpper = NegR.Upper - 1; AdjNegRUpper = NegR.Upper - 1;
PosRes = PosRes.unionWith( PosRes = PosRes.unionWith(
ConstantRange(Lo, NegL.Lower.sdiv(AdjNegRUpper - 1) + 1)); ConstantRange(Lo, NegL.Lower.sdiv(AdjNegRUpper - 1) + 1));
▲ Show 20 Lines • Show All 152 Lines • ▼ Show 20 LinesConstantRange ConstantRange::binaryXor(const ConstantRange &Other) const {
if (isEmptySet() || Other.isEmptySet()) if (isEmptySet() || Other.isEmptySet())
return getEmpty(); return getEmpty();
// Use APInt's implementation of XOR for single element ranges. // Use APInt's implementation of XOR for single element ranges.
if (isSingleElement() && Other.isSingleElement()) if (isSingleElement() && Other.isSingleElement())
return {*getSingleElement() ^ *Other.getSingleElement()}; return {*getSingleElement() ^ *Other.getSingleElement()};
// Special-case binary complement, since we can give a precise answer. // Special-case binary complement, since we can give a precise answer.
if (Other.isSingleElement() && Other.getSingleElement()->isAllOnesValue()) if (Other.isSingleElement() && Other.getSingleElement()->isAllOnes())
return binaryNot(); return binaryNot();
if (isSingleElement() && getSingleElement()->isAllOnesValue()) if (isSingleElement() && getSingleElement()->isAllOnes())
return Other.binaryNot(); return Other.binaryNot();
// TODO: replace this with something less conservative // TODO: replace this with something less conservative
return getFull(); return getFull();
} }
ConstantRange ConstantRange
ConstantRange::shl(const ConstantRange &Other) const { ConstantRange::shl(const ConstantRange &Other) const {
▲ Show 20 Lines • Show All 373 Lines • Show Last 20 Lines

llvm/lib/IR/Constants.cpp

Show First 20 Lines • Show All 89 Lines • ▼ Show 20 Lines
bool Constant::isAllOnesValue() const { bool Constant::isAllOnesValue() const {
// Check for -1 integers // Check for -1 integers
if (const ConstantInt *CI = dyn_cast<ConstantInt>(this)) if (const ConstantInt *CI = dyn_cast<ConstantInt>(this))
return CI->isMinusOne(); return CI->isMinusOne();
// Check for FP which are bitcasted from -1 integers // Check for FP which are bitcasted from -1 integers
if (const ConstantFP *CFP = dyn_cast<ConstantFP>(this)) if (const ConstantFP *CFP = dyn_cast<ConstantFP>(this))
return CFP->getValueAPF().bitcastToAPInt().isAllOnesValue(); return CFP->getValueAPF().bitcastToAPInt().isAllOnes();
// Check for constant splat vectors of 1 values. // Check for constant splat vectors of 1 values.
if (getType()->isVectorTy()) if (getType()->isVectorTy())
if (const auto *SplatVal = getSplatValue()) if (const auto *SplatVal = getSplatValue())
return SplatVal->isAllOnesValue(); return SplatVal->isAllOnesValue();
return false; return false;
} }
bool Constant::isOneValue() const { bool Constant::isOneValue() const {
// Check for 1 integers // Check for 1 integers
if (const ConstantInt *CI = dyn_cast<ConstantInt>(this)) if (const ConstantInt *CI = dyn_cast<ConstantInt>(this))
return CI->isOne(); return CI->isOne();
// Check for FP which are bitcasted from 1 integers // Check for FP which are bitcasted from 1 integers
if (const ConstantFP *CFP = dyn_cast<ConstantFP>(this)) if (const ConstantFP *CFP = dyn_cast<ConstantFP>(this))
return CFP->getValueAPF().bitcastToAPInt().isOneValue(); return CFP->getValueAPF().bitcastToAPInt().isOne();
// Check for constant splat vectors of 1 values. // Check for constant splat vectors of 1 values.
if (getType()->isVectorTy()) if (getType()->isVectorTy())
if (const auto *SplatVal = getSplatValue()) if (const auto *SplatVal = getSplatValue())
return SplatVal->isOneValue(); return SplatVal->isOneValue();
return false; return false;
} }
bool Constant::isNotOneValue() const { bool Constant::isNotOneValue() const {
// Check for 1 integers // Check for 1 integers
if (const ConstantInt *CI = dyn_cast<ConstantInt>(this)) if (const ConstantInt *CI = dyn_cast<ConstantInt>(this))
return !CI->isOneValue(); return !CI->isOneValue();
// Check for FP which are bitcasted from 1 integers // Check for FP which are bitcasted from 1 integers
if (const ConstantFP *CFP = dyn_cast<ConstantFP>(this)) if (const ConstantFP *CFP = dyn_cast<ConstantFP>(this))
return !CFP->getValueAPF().bitcastToAPInt().isOneValue(); return !CFP->getValueAPF().bitcastToAPInt().isOne();
// Check that vectors don't contain 1 // Check that vectors don't contain 1
if (auto *VTy = dyn_cast<FixedVectorType>(getType())) { if (auto *VTy = dyn_cast<FixedVectorType>(getType())) {
for (unsigned I = 0, E = VTy->getNumElements(); I != E; ++I) { for (unsigned I = 0, E = VTy->getNumElements(); I != E; ++I) {
Constant *Elt = getAggregateElement(I); Constant *Elt = getAggregateElement(I);
if (!Elt || !Elt->isNotOneValue()) if (!Elt || !Elt->isNotOneValue())
return false; return false;
} }
▲ Show 20 Lines • Show All 3,392 Lines • Show Last 20 Lines

llvm/lib/IR/Instructions.cpp

Show First 20 Lines • Show All 2,324 Lines • ▼ Show 20 Lines if (M < NumSrcElts) {
Src0Elts.setBit(i); Src0Elts.setBit(i);
Src0Identity &= (M == i); Src0Identity &= (M == i);
continue; continue;
} }
Src1Elts.setBit(i); Src1Elts.setBit(i);
Src1Identity &= (M == (i + NumSrcElts)); Src1Identity &= (M == (i + NumSrcElts));
continue; continue;
} }
assert((Src0Elts | Src1Elts | UndefElts).isAllOnesValue() && assert((Src0Elts | Src1Elts | UndefElts).isAllOnes() &&
"unknown shuffle elements"); "unknown shuffle elements");
assert(!Src0Elts.isNullValue() && !Src1Elts.isNullValue() && assert(!Src0Elts.isZero() && !Src1Elts.isZero() &&
"2-source shuffle not found"); "2-source shuffle not found");
// Determine lo/hi span ranges. // Determine lo/hi span ranges.
// TODO: How should we handle undefs at the start of subvector insertions? // TODO: How should we handle undefs at the start of subvector insertions?
int Src0Lo = Src0Elts.countTrailingZeros(); int Src0Lo = Src0Elts.countTrailingZeros();
int Src1Lo = Src1Elts.countTrailingZeros(); int Src1Lo = Src1Elts.countTrailingZeros();
int Src0Hi = NumMaskElts - Src0Elts.countLeadingZeros(); int Src0Hi = NumMaskElts - Src0Elts.countLeadingZeros();
int Src1Hi = NumMaskElts - Src1Elts.countLeadingZeros(); int Src1Hi = NumMaskElts - Src1Elts.countLeadingZeros();
▲ Show 20 Lines • Show All 2,324 Lines • Show Last 20 Lines

llvm/lib/IR/Verifier.cpp

Show First 20 Lines • Show All 5,064 Lines • ▼ Show 20 Lines Assert(cast<VectorType>(Mask->getType())->getElementCount() ==
cast<VectorType>(Val->getType())->getElementCount(), cast<VectorType>(Val->getType())->getElementCount(),
"masked_store: vector mask must be same length as value", Call); "masked_store: vector mask must be same length as value", Call);
break; break;
} }
case Intrinsic::masked_gather: { case Intrinsic::masked_gather: {
const APInt &Alignment = const APInt &Alignment =
cast<ConstantInt>(Call.getArgOperand(1))->getValue(); cast<ConstantInt>(Call.getArgOperand(1))->getValue();
Assert(Alignment.isNullValue() || Alignment.isPowerOf2(), Assert(Alignment.isZero() || Alignment.isPowerOf2(),
"masked_gather: alignment must be 0 or a power of 2", Call); "masked_gather: alignment must be 0 or a power of 2", Call);
break; break;
} }
case Intrinsic::masked_scatter: { case Intrinsic::masked_scatter: {
const APInt &Alignment = const APInt &Alignment =
cast<ConstantInt>(Call.getArgOperand(2))->getValue(); cast<ConstantInt>(Call.getArgOperand(2))->getValue();
Assert(Alignment.isNullValue() || Alignment.isPowerOf2(), Assert(Alignment.isZero() || Alignment.isPowerOf2(),
"masked_scatter: alignment must be 0 or a power of 2", Call); "masked_scatter: alignment must be 0 or a power of 2", Call);
break; break;
} }
case Intrinsic::experimental_guard: { case Intrinsic::experimental_guard: {
Assert(isa<CallInst>(Call), "experimental_guard cannot be invoked", Call); Assert(isa<CallInst>(Call), "experimental_guard cannot be invoked", Call);
Assert(Call.countOperandBundlesOfType(LLVMContext::OB_deopt) == 1, Assert(Call.countOperandBundlesOfType(LLVMContext::OB_deopt) == 1,
"experimental_guard must have exactly one " "experimental_guard must have exactly one "
▲ Show 20 Lines • Show All 1,186 Lines • Show Last 20 Lines

llvm/lib/Support/APFixedPoint.cpp

Show First 20 Lines • Show All 300 Lines • ▼ Show 20 LinesAPFixedPoint APFixedPoint::div(const APFixedPoint &Other,
// perform the full division. // perform the full division.
ThisVal = ThisVal.shl(CommonFXSema.getScale()); ThisVal = ThisVal.shl(CommonFXSema.getScale());
APSInt Result; APSInt Result;
if (CommonFXSema.isSigned()) { if (CommonFXSema.isSigned()) {
APInt Rem; APInt Rem;
APInt::sdivrem(ThisVal, OtherVal, Result, Rem); APInt::sdivrem(ThisVal, OtherVal, Result, Rem);
// If the quotient is negative and the remainder is nonzero, round // If the quotient is negative and the remainder is nonzero, round
// towards negative infinity by subtracting epsilon from the result. // towards negative infinity by subtracting epsilon from the result.
if (ThisVal.isNegative() != OtherVal.isNegative() && !Rem.isNullValue()) if (ThisVal.isNegative() != OtherVal.isNegative() && !Rem.isZero())
Result = Result - 1; Result = Result - 1;
} else } else
Result = ThisVal.udiv(OtherVal); Result = ThisVal.udiv(OtherVal);
Result.setIsSigned(CommonFXSema.isSigned()); Result.setIsSigned(CommonFXSema.isSigned());
// If our result lies outside of the representative range of the common // If our result lies outside of the representative range of the common
// semantic, we either have overflow or saturation. // semantic, we either have overflow or saturation.
APSInt Max = APFixedPoint::getMax(CommonFXSema).getValue() APSInt Max = APFixedPoint::getMax(CommonFXSema).getValue()
▲ Show 20 Lines • Show All 257 Lines • Show Last 20 Lines

llvm/lib/Support/APInt.cpp

Show First 20 Lines • Show All 1,937 Lines • ▼ Show 20 Lines
APInt APInt::usub_ov(const APInt &RHS, bool &Overflow) const { APInt APInt::usub_ov(const APInt &RHS, bool &Overflow) const {
APInt Res = *this-RHS; APInt Res = *this-RHS;
Overflow = Res.ugt(*this); Overflow = Res.ugt(*this);
return Res; return Res;
} }
APInt APInt::sdiv_ov(const APInt &RHS, bool &Overflow) const { APInt APInt::sdiv_ov(const APInt &RHS, bool &Overflow) const {
// MININT/-1 --> overflow. // MININT/-1 --> overflow.
Overflow = isMinSignedValue() && RHS.isAllOnesValue(); Overflow = isMinSignedValue() && RHS.isAllOnes();
return sdiv(RHS); return sdiv(RHS);
} }
APInt APInt::smul_ov(const APInt &RHS, bool &Overflow) const { APInt APInt::smul_ov(const APInt &RHS, bool &Overflow) const {
APInt Res = *this * RHS; APInt Res = *this * RHS;
if (*this != 0 && RHS != 0) if (*this != 0 && RHS != 0)
Overflow = Res.sdiv(RHS) != *this || Res.sdiv(*this) != RHS; Overflow = Res.sdiv(RHS) != *this || Res.sdiv(*this) != RHS;
▲ Show 20 Lines • Show All 1,010 Lines • ▼ Show 20 Lines assert((((OldBitWidth % NewBitWidth) == 0) ||
((NewBitWidth % OldBitWidth) == 0)) && ((NewBitWidth % OldBitWidth) == 0)) &&
"One size should be a multiple of the other one. " "One size should be a multiple of the other one. "
"Can't do fractional scaling."); "Can't do fractional scaling.");
// Check for matching bitwidths. // Check for matching bitwidths.
if (OldBitWidth == NewBitWidth) if (OldBitWidth == NewBitWidth)
return A; return A;
APInt NewA = APInt::getNullValue(NewBitWidth); APInt NewA = APInt::getZero(NewBitWidth);
// Check for null input. // Check for null input.
if (A.isNullValue()) if (A.isZero())
return NewA; return NewA;
if (NewBitWidth > OldBitWidth) { if (NewBitWidth > OldBitWidth) {
// Repeat bits. // Repeat bits.
unsigned Scale = NewBitWidth / OldBitWidth; unsigned Scale = NewBitWidth / OldBitWidth;
for (unsigned i = 0; i != OldBitWidth; ++i) for (unsigned i = 0; i != OldBitWidth; ++i)
if (A[i]) if (A[i])
NewA.setBits(i * Scale, (i + 1) * Scale); NewA.setBits(i * Scale, (i + 1) * Scale);
} else { } else {
// Merge bits - if any old bit is set, then set scale equivalent new bit. // Merge bits - if any old bit is set, then set scale equivalent new bit.
unsigned Scale = OldBitWidth / NewBitWidth; unsigned Scale = OldBitWidth / NewBitWidth;
for (unsigned i = 0; i != NewBitWidth; ++i) for (unsigned i = 0; i != NewBitWidth; ++i)
if (!A.extractBits(Scale, i * Scale).isNullValue()) if (!A.extractBits(Scale, i * Scale).isZero())
NewA.setBit(i); NewA.setBit(i);
} }
return NewA; return NewA;
} }
/// StoreIntToMemory - Fills the StoreBytes bytes of memory starting from Dst /// StoreIntToMemory - Fills the StoreBytes bytes of memory starting from Dst
/// with the integer held in IntVal. /// with the integer held in IntVal.
▲ Show 20 Lines • Show All 51 Lines • Show Last 20 Lines

llvm/lib/Support/KnownBits.cpp

Show First 20 Lines • Show All 398 Lines • ▼ Show 20 LinesKnownBits KnownBits::abs(bool IntMinIsPoison) const {
// Absolute value preserves trailing zero count. // Absolute value preserves trailing zero count.
KnownBits KnownAbs(getBitWidth()); KnownBits KnownAbs(getBitWidth());
KnownAbs.Zero.setLowBits(countMinTrailingZeros()); KnownAbs.Zero.setLowBits(countMinTrailingZeros());
// We only know that the absolute values's MSB will be zero if INT_MIN is // We only know that the absolute values's MSB will be zero if INT_MIN is
// poison, or there is a set bit that isn't the sign bit (otherwise it could // poison, or there is a set bit that isn't the sign bit (otherwise it could
// be INT_MIN). // be INT_MIN).
if (IntMinIsPoison || (!One.isNullValue() && !One.isMinSignedValue())) if (IntMinIsPoison || (!One.isZero() && !One.isMinSignedValue()))
KnownAbs.Zero.setSignBit(); KnownAbs.Zero.setSignBit();
// FIXME: Handle known negative input? // FIXME: Handle known negative input?
// FIXME: Calculate the negated Known bits and combine them? // FIXME: Calculate the negated Known bits and combine them?
return KnownAbs; return KnownAbs;
} }
KnownBits KnownBits::mul(const KnownBits &LHS, const KnownBits &RHS, KnownBits KnownBits::mul(const KnownBits &LHS, const KnownBits &RHS,
▲ Show 20 Lines • Show All 210 Lines • Show Last 20 Lines

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

Show First 20 Lines • Show All 2,161 Lines • ▼ Show 20 Linesstatic bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted,
assert((VT == MVT::i32 || VT == MVT::i64) && assert((VT == MVT::i32 || VT == MVT::i64) &&
"i32 or i64 mask type expected!"); "i32 or i64 mask type expected!");
unsigned BitWidth = VT.getSizeInBits() - NumberOfIgnoredHighBits; unsigned BitWidth = VT.getSizeInBits() - NumberOfIgnoredHighBits;
APInt SignificantDstMask = APInt(BitWidth, DstMask); APInt SignificantDstMask = APInt(BitWidth, DstMask);
APInt SignificantBitsToBeInserted = BitsToBeInserted.zextOrTrunc(BitWidth); APInt SignificantBitsToBeInserted = BitsToBeInserted.zextOrTrunc(BitWidth);
return (SignificantDstMask & SignificantBitsToBeInserted) == 0 && return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
(SignificantDstMask | SignificantBitsToBeInserted).isAllOnesValue(); (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
} }
// Look for bits that will be useful for later uses. // Look for bits that will be useful for later uses.
// A bit is consider useless as soon as it is dropped and never used // A bit is consider useless as soon as it is dropped and never used
// before it as been dropped. // before it as been dropped.
// E.g., looking for useful bit of x // E.g., looking for useful bit of x
// 1. y = x & 0x7 // 1. y = x & 0x7
// 2. z = y >> 2 // 2. z = y >> 2
▲ Show 20 Lines • Show All 2,896 Lines • Show Last 20 Lines

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 10,218 Lines • ▼ Show 20 Lines if (VT.isInteger()) {
// patterns for these operations to match, which will lower these constants // patterns for these operations to match, which will lower these constants
// to whatever is proven necessary. // to whatever is proven necessary.
BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode()); BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
if (BVN->isConstant()) if (BVN->isConstant())
if (ConstantSDNode *Const = BVN->getConstantSplatNode()) { if (ConstantSDNode *Const = BVN->getConstantSplatNode()) {
unsigned BitSize = VT.getVectorElementType().getSizeInBits(); unsigned BitSize = VT.getVectorElementType().getSizeInBits();
APInt Val(BitSize, APInt Val(BitSize,
Const->getAPIntValue().zextOrTrunc(BitSize).getZExtValue()); Const->getAPIntValue().zextOrTrunc(BitSize).getZExtValue());
if (Val.isNullValue() || Val.isAllOnesValue()) if (Val.isZero() || Val.isAllOnes())
return Op; return Op;
} }
} }
if (SDValue V = ConstantBuildVector(Op, DAG)) if (SDValue V = ConstantBuildVector(Op, DAG))
return V; return V;
// Scan through the operands to find some interesting properties we can // Scan through the operands to find some interesting properties we can
▲ Show 20 Lines • Show All 5,910 Lines • ▼ Show 20 Lines if (SetCC.getOpcode() == ISD::SETCC &&
APInt SplatLHSVal; APInt SplatLHSVal;
if (CmpLHS.getValueType() == N->getOperand(1).getValueType() && if (CmpLHS.getValueType() == N->getOperand(1).getValueType() &&
VT.isSimple() && VT.isSimple() &&
is_contained( is_contained(
makeArrayRef({MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, makeArrayRef({MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
MVT::v2i32, MVT::v4i32, MVT::v2i64}), MVT::v2i32, MVT::v4i32, MVT::v2i64}),
VT.getSimpleVT().SimpleTy) && VT.getSimpleVT().SimpleTy) &&
ISD::isConstantSplatVector(SplatLHS, SplatLHSVal) && ISD::isConstantSplatVector(SplatLHS, SplatLHSVal) &&
SplatLHSVal.isOneValue() && ISD::isConstantSplatVectorAllOnes(CmpRHS) && SplatLHSVal.isOne() && ISD::isConstantSplatVectorAllOnes(CmpRHS) &&
ISD::isConstantSplatVectorAllOnes(SplatRHS)) { ISD::isConstantSplatVectorAllOnes(SplatRHS)) {
unsigned NumElts = VT.getVectorNumElements(); unsigned NumElts = VT.getVectorNumElements();
SmallVector<SDValue, 8> Ops( SmallVector<SDValue, 8> Ops(
NumElts, DAG.getConstant(VT.getScalarSizeInBits() - 1, SDLoc(N), NumElts, DAG.getConstant(VT.getScalarSizeInBits() - 1, SDLoc(N),
VT.getScalarType())); VT.getScalarType()));
SDValue Val = DAG.getBuildVector(VT, SDLoc(N), Ops); SDValue Val = DAG.getBuildVector(VT, SDLoc(N), Ops);
auto Shift = DAG.getNode(ISD::SRA, SDLoc(N), VT, CmpLHS, Val); auto Shift = DAG.getNode(ISD::SRA, SDLoc(N), VT, CmpLHS, Val);
▲ Show 20 Lines • Show All 2,844 Lines • Show Last 20 Lines

llvm/lib/Target/ARM/ARMISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 7,710 Lines • ▼ Show 20 LinesSDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
if (SDValue R = LowerBUILD_VECTORToVIDUP(Op, DAG, ST)) if (SDValue R = LowerBUILD_VECTORToVIDUP(Op, DAG, ST))
return R; return R;
APInt SplatBits, SplatUndef; APInt SplatBits, SplatUndef;
unsigned SplatBitSize; unsigned SplatBitSize;
bool HasAnyUndefs; bool HasAnyUndefs;
if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) { if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
if (SplatUndef.isAllOnesValue()) if (SplatUndef.isAllOnes())
return DAG.getUNDEF(VT); return DAG.getUNDEF(VT);
if ((ST->hasNEON() && SplatBitSize <= 64) || if ((ST->hasNEON() && SplatBitSize <= 64) ||
(ST->hasMVEIntegerOps() && SplatBitSize <= 64)) { (ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
// Check if an immediate VMOV works. // Check if an immediate VMOV works.
EVT VmovVT; EVT VmovVT;
SDValue Val = SDValue Val =
isVMOVModifiedImm(SplatBits.getZExtValue(), SplatUndef.getZExtValue(), isVMOVModifiedImm(SplatBits.getZExtValue(), SplatUndef.getZExtValue(),
▲ Show 20 Lines • Show All 13,393 Lines • Show Last 20 Lines

llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp

Show First 20 Lines • Show All 169 Lines • ▼ Show 20 Lines if (match(Arg, PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>(
return IC.replaceInstUsesWith(II, ArgArg); return IC.replaceInstUsesWith(II, ArgArg);
} }
Constant *XorMask; Constant *XorMask;
if (match(Arg, m_Xor(PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>( if (match(Arg, m_Xor(PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>(
PatternMatch::m_Value(ArgArg)), PatternMatch::m_Value(ArgArg)),
PatternMatch::m_Constant(XorMask))) && PatternMatch::m_Constant(XorMask))) &&
II.getType() == ArgArg->getType()) { II.getType() == ArgArg->getType()) {
if (auto *CI = dyn_cast<ConstantInt>(XorMask)) { if (auto *CI = dyn_cast<ConstantInt>(XorMask)) {
if (CI->getValue().trunc(16).isAllOnesValue()) { if (CI->getValue().trunc(16).isAllOnes()) {
auto TrueVector = IC.Builder.CreateVectorSplat( auto TrueVector = IC.Builder.CreateVectorSplat(
cast<FixedVectorType>(II.getType())->getNumElements(), cast<FixedVectorType>(II.getType())->getNumElements(),
IC.Builder.getTrue()); IC.Builder.getTrue());
return BinaryOperator::Create(Instruction::Xor, ArgArg, TrueVector); return BinaryOperator::Create(Instruction::Xor, ArgArg, TrueVector);
} }
} }
} }
KnownBits ScalarKnown(32); KnownBits ScalarKnown(32);
▲ Show 20 Lines • Show All 218 Lines • ▼ Show 20 Lines if (ST->isThumb2() && NegImm < 1<<12)
// icmp X, #-C -> cmn X, #C // icmp X, #-C -> cmn X, #C
return 0; return 0;
if (ST->isThumb() && NegImm < 1<<8) if (ST->isThumb() && NegImm < 1<<8)
// icmp X, #-C -> adds X, #C // icmp X, #-C -> adds X, #C
return 0; return 0;
} }
// xor a, -1 can always be folded to MVN // xor a, -1 can always be folded to MVN
if (Opcode == Instruction::Xor && Imm.isAllOnesValue()) if (Opcode == Instruction::Xor && Imm.isAllOnes())
return 0; return 0;
// Ensures negative constant of min(max()) or max(min()) patterns that // Ensures negative constant of min(max()) or max(min()) patterns that
// match to SSAT instructions don't get hoisted // match to SSAT instructions don't get hoisted
if (Inst && ((ST->hasV6Ops() && !ST->isThumb()) || ST->isThumb2()) && if (Inst && ((ST->hasV6Ops() && !ST->isThumb()) || ST->isThumb2()) &&
Ty->getIntegerBitWidth() <= 32) { Ty->getIntegerBitWidth() <= 32) {
if (isSSATMinMaxPattern(Inst, Imm) || if (isSSATMinMaxPattern(Inst, Imm) ||
(isa<ICmpInst>(Inst) && Inst->hasOneUse() && (isa<ICmpInst>(Inst) && Inst->hasOneUse() &&
▲ Show 20 Lines • Show All 1,918 Lines • Show Last 20 Lines

llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp

Show First 20 Lines • Show All 1,342 Lines • ▼ Show 20 Lines#define CallBuilder(B, F) \
// First, try to simplify the subtraction directly. // First, try to simplify the subtraction directly.
if (auto *Diff = dyn_cast<ConstantInt>( if (auto *Diff = dyn_cast<ConstantInt>(
Simplify(CallBuilder(B, CreateSub(Idx0, Idx1))))) Simplify(CallBuilder(B, CreateSub(Idx0, Idx1)))))
return Diff->getSExtValue() * Scale; return Diff->getSExtValue() * Scale;
KnownBits Known0 = computeKnownBits(Idx0, DL, 0, &AC, Gep0, &DT); KnownBits Known0 = computeKnownBits(Idx0, DL, 0, &AC, Gep0, &DT);
KnownBits Known1 = computeKnownBits(Idx1, DL, 0, &AC, Gep1, &DT); KnownBits Known1 = computeKnownBits(Idx1, DL, 0, &AC, Gep1, &DT);
APInt Unknown = ~(Known0.Zero | Known0.One) | ~(Known1.Zero | Known1.One); APInt Unknown = ~(Known0.Zero | Known0.One) | ~(Known1.Zero | Known1.One);
if (Unknown.isAllOnesValue()) if (Unknown.isAllOnes())
return None; return None;
Value *MaskU = ConstantInt::get(Idx0->getType(), Unknown); Value *MaskU = ConstantInt::get(Idx0->getType(), Unknown);
Value *AndU0 = Simplify(CallBuilder(B, CreateAnd(Idx0, MaskU))); Value *AndU0 = Simplify(CallBuilder(B, CreateAnd(Idx0, MaskU)));
Value *AndU1 = Simplify(CallBuilder(B, CreateAnd(Idx1, MaskU))); Value *AndU1 = Simplify(CallBuilder(B, CreateAnd(Idx1, MaskU)));
Value *SubU = Simplify(CallBuilder(B, CreateSub(AndU0, AndU1))); Value *SubU = Simplify(CallBuilder(B, CreateSub(AndU0, AndU1)));
int Diff0 = 0; int Diff0 = 0;
if (auto *C = dyn_cast<ConstantInt>(SubU)) { if (auto *C = dyn_cast<ConstantInt>(SubU)) {
▲ Show 20 Lines • Show All 170 Lines • Show Last 20 Lines

llvm/lib/Target/Mips/MipsInstructionSelector.cpp

Show First 20 Lines • Show All 139 Lines • ▼ Show 20 Linesconst TargetRegisterClass *MipsInstructionSelector::getRegClassForTypeOnBank(
llvm_unreachable("Unsupported register bank."); llvm_unreachable("Unsupported register bank.");
} }
bool MipsInstructionSelector::materialize32BitImm(Register DestReg, APInt Imm, bool MipsInstructionSelector::materialize32BitImm(Register DestReg, APInt Imm,
MachineIRBuilder &B) const { MachineIRBuilder &B) const {
assert(Imm.getBitWidth() == 32 && "Unsupported immediate size."); assert(Imm.getBitWidth() == 32 && "Unsupported immediate size.");
// Ori zero extends immediate. Used for values with zeros in high 16 bits. // Ori zero extends immediate. Used for values with zeros in high 16 bits.
if (Imm.getHiBits(16).isNullValue()) { if (Imm.getHiBits(16).isZero()) {
MachineInstr *Inst = MachineInstr *Inst =
B.buildInstr(Mips::ORi, {DestReg}, {Register(Mips::ZERO)}) B.buildInstr(Mips::ORi, {DestReg}, {Register(Mips::ZERO)})
.addImm(Imm.getLoBits(16).getLimitedValue()); .addImm(Imm.getLoBits(16).getLimitedValue());
return constrainSelectedInstRegOperands(*Inst, TII, TRI, RBI); return constrainSelectedInstRegOperands(*Inst, TII, TRI, RBI);
} }
// Lui places immediate in high 16 bits and sets low 16 bits to zero. // Lui places immediate in high 16 bits and sets low 16 bits to zero.
if (Imm.getLoBits(16).isNullValue()) { if (Imm.getLoBits(16).isZero()) {
MachineInstr *Inst = B.buildInstr(Mips::LUi, {DestReg}, {}) MachineInstr *Inst = B.buildInstr(Mips::LUi, {DestReg}, {})
.addImm(Imm.getHiBits(16).getLimitedValue()); .addImm(Imm.getHiBits(16).getLimitedValue());
return constrainSelectedInstRegOperands(*Inst, TII, TRI, RBI); return constrainSelectedInstRegOperands(*Inst, TII, TRI, RBI);
} }
// ADDiu sign extends immediate. Used for values with 1s in high 17 bits. // ADDiu sign extends immediate. Used for values with 1s in high 17 bits.
if (Imm.isSignedIntN(16)) { if (Imm.isSignedIntN(16)) {
MachineInstr *Inst = MachineInstr *Inst =
B.buildInstr(Mips::ADDiu, {DestReg}, {Register(Mips::ZERO)}) B.buildInstr(Mips::ADDiu, {DestReg}, {Register(Mips::ZERO)})
▲ Show 20 Lines • Show All 777 Lines • Show Last 20 Lines

llvm/lib/Target/Mips/MipsSEISelLowering.cpp

Show First 20 Lines • Show All 563 Lines • ▼ Show 20 Linesstatic bool isVectorAllOnes(SDValue N) {
APInt SplatValue, SplatUndef; APInt SplatValue, SplatUndef;
unsigned SplatBitSize; unsigned SplatBitSize;
bool HasAnyUndefs; bool HasAnyUndefs;
// Endianness doesn't matter in this context because we are looking for // Endianness doesn't matter in this context because we are looking for
// an all-ones value. // an all-ones value.
if (BVN->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs)) if (BVN->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs))
return SplatValue.isAllOnesValue(); return SplatValue.isAllOnes();
return false; return false;
} }
// Test whether N is the bitwise inverse of OfNode. // Test whether N is the bitwise inverse of OfNode.
static bool isBitwiseInverse(SDValue N, SDValue OfNode) { static bool isBitwiseInverse(SDValue N, SDValue OfNode) {
if (N->getOpcode() != ISD::XOR) if (N->getOpcode() != ISD::XOR)
return false; return false;
▲ Show 20 Lines • Show All 115 Lines • ▼ Show 20 Lines if (Op0->getOpcode() == ISD::AND && Op1->getOpcode() == ISD::AND) {
// At this point, IfClr will be set if we have a valid match. // At this point, IfClr will be set if we have a valid match.
if (!IfClr.getNode()) if (!IfClr.getNode())
return SDValue(); return SDValue();
assert(Cond.getNode() && IfSet.getNode()); assert(Cond.getNode() && IfSet.getNode());
// Fold degenerate cases. // Fold degenerate cases.
if (IsConstantMask) { if (IsConstantMask) {
if (Mask.isAllOnesValue()) if (Mask.isAllOnes())
return IfSet; return IfSet;
else if (Mask == 0) else if (Mask == 0)
return IfClr; return IfClr;
} }
// Transform the DAG into an equivalent VSELECT. // Transform the DAG into an equivalent VSELECT.
return DAG.getNode(ISD::VSELECT, SDLoc(N), Ty, Cond, IfSet, IfClr); return DAG.getNode(ISD::VSELECT, SDLoc(N), Ty, Cond, IfSet, IfClr);
} }
▲ Show 20 Lines • Show All 3,154 Lines • Show Last 20 Lines

llvm/lib/Target/PowerPC/PPCInstrInfo.cpp

Show First 20 Lines • Show All 3,773 Lines • ▼ Show 20 Linesbool PPCInstrInfo::combineRLWINM(MachineInstr &MI,
APInt MaskSrc = APInt::getBitsSetWithWrap(32, 32 - MESrc - 1, 32 - MBSrc); APInt MaskSrc = APInt::getBitsSetWithWrap(32, 32 - MESrc - 1, 32 - MBSrc);
APInt RotatedSrcMask = MaskSrc.rotl(SHMI); APInt RotatedSrcMask = MaskSrc.rotl(SHMI);
APInt FinalMask = RotatedSrcMask & MaskMI; APInt FinalMask = RotatedSrcMask & MaskMI;
uint32_t NewMB, NewME; uint32_t NewMB, NewME;
bool Simplified = false; bool Simplified = false;
// If final mask is 0, MI result should be 0 too. // If final mask is 0, MI result should be 0 too.
if (FinalMask.isNullValue()) { if (FinalMask.isZero()) {
bool Is64Bit = bool Is64Bit =
(MI.getOpcode() == PPC::RLWINM8 || MI.getOpcode() == PPC::RLWINM8_rec); (MI.getOpcode() == PPC::RLWINM8 || MI.getOpcode() == PPC::RLWINM8_rec);
Simplified = true; Simplified = true;
LLVM_DEBUG(dbgs() << "Replace Instr: "); LLVM_DEBUG(dbgs() << "Replace Instr: ");
LLVM_DEBUG(MI.dump()); LLVM_DEBUG(MI.dump());
if (MI.getOpcode() == PPC::RLWINM || MI.getOpcode() == PPC::RLWINM8) { if (MI.getOpcode() == PPC::RLWINM || MI.getOpcode() == PPC::RLWINM8) {
// Replace MI with "LI 0" // Replace MI with "LI 0"
▲ Show 20 Lines • Show All 1,735 Lines • Show Last 20 Lines

llvm/lib/Target/X86/X86ISelDAGToDAG.cpp

Show First 20 Lines • Show All 884 Lines • ▼ Show 20 Lines for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
/// The all-ones vector constant can be materialized using a pcmpeq /// The all-ones vector constant can be materialized using a pcmpeq
/// instruction that is commonly recognized as an idiom (has no register /// instruction that is commonly recognized as an idiom (has no register
/// dependency), so that's better/smaller than loading a splat 1 constant. /// dependency), so that's better/smaller than loading a splat 1 constant.
if ((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && if ((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
N->getSimpleValueType(0).isVector()) { N->getSimpleValueType(0).isVector()) {
APInt SplatVal; APInt SplatVal;
if (X86::isConstantSplat(N->getOperand(1), SplatVal) && if (X86::isConstantSplat(N->getOperand(1), SplatVal) &&
SplatVal.isOneValue()) { SplatVal.isOne()) {
SDLoc DL(N); SDLoc DL(N);
MVT VT = N->getSimpleValueType(0); MVT VT = N->getSimpleValueType(0);
unsigned NumElts = VT.getSizeInBits() / 32; unsigned NumElts = VT.getSizeInBits() / 32;
SDValue AllOnes = SDValue AllOnes =
CurDAG->getAllOnesConstant(DL, MVT::getVectorVT(MVT::i32, NumElts)); CurDAG->getAllOnesConstant(DL, MVT::getVectorVT(MVT::i32, NumElts));
AllOnes = CurDAG->getBitcast(VT, AllOnes); AllOnes = CurDAG->getBitcast(VT, AllOnes);
▲ Show 20 Lines • Show All 3,449 Lines • ▼ Show 20 Linesbool X86DAGToDAGISel::shrinkAndImmediate(SDNode *And) {
// The variable operand must be all zeros in the top bits to allow using the // The variable operand must be all zeros in the top bits to allow using the
// new, negative constant as the mask. // new, negative constant as the mask.
if (!CurDAG->MaskedValueIsZero(And0, HighZeros)) if (!CurDAG->MaskedValueIsZero(And0, HighZeros))
return false; return false;
// Check if the mask is -1. In that case, this is an unnecessary instruction // Check if the mask is -1. In that case, this is an unnecessary instruction
// that escaped earlier analysis. // that escaped earlier analysis.
if (NegMaskVal.isAllOnesValue()) { if (NegMaskVal.isAllOnes()) {
ReplaceNode(And, And0.getNode()); ReplaceNode(And, And0.getNode());
return true; return true;
} }
// A negative mask allows a smaller encoding. Create a new 'and' node. // A negative mask allows a smaller encoding. Create a new 'and' node.
SDValue NewMask = CurDAG->getConstant(NegMaskVal, SDLoc(And), VT); SDValue NewMask = CurDAG->getConstant(NegMaskVal, SDLoc(And), VT);
insertDAGNode(*CurDAG, SDValue(And, 0), NewMask); insertDAGNode(*CurDAG, SDValue(And, 0), NewMask);
SDValue NewAnd = CurDAG->getNode(ISD::AND, SDLoc(And), VT, And0, NewMask); SDValue NewAnd = CurDAG->getNode(ISD::AND, SDLoc(And), VT, And0, NewMask);
▲ Show 20 Lines • Show All 1,700 Lines • Show Last 20 Lines

llvm/lib/Target/X86/X86ISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 5,948 Lines • ▼ Show 20 Lines
static bool canWidenShuffleElements(ArrayRef<int> Mask, static bool canWidenShuffleElements(ArrayRef<int> Mask,
const APInt &Zeroable, const APInt &Zeroable,
bool V2IsZero, bool V2IsZero,
SmallVectorImpl<int> &WidenedMask) { SmallVectorImpl<int> &WidenedMask) {
// Create an alternative mask with info about zeroable elements. // Create an alternative mask with info about zeroable elements.
// Here we do not set undef elements as zeroable. // Here we do not set undef elements as zeroable.
SmallVector<int, 64> ZeroableMask(Mask.begin(), Mask.end()); SmallVector<int, 64> ZeroableMask(Mask.begin(), Mask.end());
if (V2IsZero) { if (V2IsZero) {
assert(!Zeroable.isNullValue() && "V2's non-undef elements are used?!"); assert(!Zeroable.isZero() && "V2's non-undef elements are used?!");
for (int i = 0, Size = Mask.size(); i != Size; ++i) for (int i = 0, Size = Mask.size(); i != Size; ++i)
if (Mask[i] != SM_SentinelUndef && Zeroable[i]) if (Mask[i] != SM_SentinelUndef && Zeroable[i])
ZeroableMask[i] = SM_SentinelZero; ZeroableMask[i] = SM_SentinelZero;
} }
return canWidenShuffleElements(ZeroableMask, WidenedMask); return canWidenShuffleElements(ZeroableMask, WidenedMask);
} }
static bool canWidenShuffleElements(ArrayRef<int> Mask) { static bool canWidenShuffleElements(ArrayRef<int> Mask) {
▲ Show 20 Lines • Show All 822 Lines • ▼ Show 20 Lines auto CastBitData = [&](APInt &UndefSrcElts, ArrayRef<APInt> SrcEltBits) {
UndefElts = APInt(NumElts, 0); UndefElts = APInt(NumElts, 0);
EltBits.resize(NumElts, APInt(EltSizeInBits, 0)); EltBits.resize(NumElts, APInt(EltSizeInBits, 0));
for (unsigned i = 0; i != NumElts; ++i) { for (unsigned i = 0; i != NumElts; ++i) {
unsigned BitOffset = i * EltSizeInBits; unsigned BitOffset = i * EltSizeInBits;
APInt UndefEltBits = UndefBits.extractBits(EltSizeInBits, BitOffset); APInt UndefEltBits = UndefBits.extractBits(EltSizeInBits, BitOffset);
// Only treat an element as UNDEF if all bits are UNDEF. // Only treat an element as UNDEF if all bits are UNDEF.
if (UndefEltBits.isAllOnesValue()) { if (UndefEltBits.isAllOnes()) {
if (!AllowWholeUndefs) if (!AllowWholeUndefs)
return false; return false;
UndefElts.setBit(i); UndefElts.setBit(i);
continue; continue;
} }
// If only some bits are UNDEF then treat them as zero (or bail if not // If only some bits are UNDEF then treat them as zero (or bail if not
// supported). // supported).
▲ Show 20 Lines • Show All 1,185 Lines • ▼ Show 20 Lines case X86ISD::PACKUS: {
APInt EltsLHS, EltsRHS; APInt EltsLHS, EltsRHS;
getPackDemandedElts(VT, DemandedElts, EltsLHS, EltsRHS); getPackDemandedElts(VT, DemandedElts, EltsLHS, EltsRHS);
// If we know input saturation won't happen (or we don't care for particular // If we know input saturation won't happen (or we don't care for particular
// lanes), we can treat this as a truncation shuffle. // lanes), we can treat this as a truncation shuffle.
bool Offset0 = false, Offset1 = false; bool Offset0 = false, Offset1 = false;
if (Opcode == X86ISD::PACKSS) { if (Opcode == X86ISD::PACKSS) {
if ((!(N0.isUndef() || EltsLHS.isNullValue()) && if ((!(N0.isUndef() || EltsLHS.isZero()) &&
DAG.ComputeNumSignBits(N0, EltsLHS, Depth + 1) <= NumBitsPerElt) || DAG.ComputeNumSignBits(N0, EltsLHS, Depth + 1) <= NumBitsPerElt) ||
(!(N1.isUndef() || EltsRHS.isNullValue()) && (!(N1.isUndef() || EltsRHS.isZero()) &&
DAG.ComputeNumSignBits(N1, EltsRHS, Depth + 1) <= NumBitsPerElt)) DAG.ComputeNumSignBits(N1, EltsRHS, Depth + 1) <= NumBitsPerElt))
return false; return false;
// We can't easily fold ASHR into a shuffle, but if it was feeding a // We can't easily fold ASHR into a shuffle, but if it was feeding a
// PACKSS then it was likely being used for sign-extension for a // PACKSS then it was likely being used for sign-extension for a
// truncation, so just peek through and adjust the mask accordingly. // truncation, so just peek through and adjust the mask accordingly.
if (N0.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N0.getNode()) && if (N0.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N0.getNode()) &&
N0.getConstantOperandAPInt(1) == NumBitsPerElt) { N0.getConstantOperandAPInt(1) == NumBitsPerElt) {
Offset0 = true; Offset0 = true;
N0 = N0.getOperand(0); N0 = N0.getOperand(0);
} }
if (N1.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N1.getNode()) && if (N1.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N1.getNode()) &&
N1.getConstantOperandAPInt(1) == NumBitsPerElt) { N1.getConstantOperandAPInt(1) == NumBitsPerElt) {
Offset1 = true; Offset1 = true;
N1 = N1.getOperand(0); N1 = N1.getOperand(0);
} }
} else { } else {
APInt ZeroMask = APInt::getHighBitsSet(2 * NumBitsPerElt, NumBitsPerElt); APInt ZeroMask = APInt::getHighBitsSet(2 * NumBitsPerElt, NumBitsPerElt);
if ((!(N0.isUndef() || EltsLHS.isNullValue()) && if ((!(N0.isUndef() || EltsLHS.isZero()) &&
!DAG.MaskedValueIsZero(N0, ZeroMask, EltsLHS, Depth + 1)) || !DAG.MaskedValueIsZero(N0, ZeroMask, EltsLHS, Depth + 1)) ||
(!(N1.isUndef() || EltsRHS.isNullValue()) && (!(N1.isUndef() || EltsRHS.isZero()) &&
!DAG.MaskedValueIsZero(N1, ZeroMask, EltsRHS, Depth + 1))) !DAG.MaskedValueIsZero(N1, ZeroMask, EltsRHS, Depth + 1)))
return false; return false;
} }
bool IsUnary = (N0 == N1); bool IsUnary = (N0 == N1);
Ops.push_back(N0); Ops.push_back(N0);
if (!IsUnary) if (!IsUnary)
▲ Show 20 Lines • Show All 872 Lines • ▼ Show 20 Lines if (!IsAfterLegalize && VT.isVector()) {
return DAG.getVectorShuffle(VT, DL, V, Z, ClearMask); return DAG.getVectorShuffle(VT, DL, V, Z, ClearMask);
} }
} }
} }
// If the upper half of a ymm/zmm load is undef then just load the lower half. // If the upper half of a ymm/zmm load is undef then just load the lower half.
if (VT.is256BitVector() || VT.is512BitVector()) { if (VT.is256BitVector() || VT.is512BitVector()) {
unsigned HalfNumElems = NumElems / 2; unsigned HalfNumElems = NumElems / 2;
if (UndefMask.extractBits(HalfNumElems, HalfNumElems).isAllOnesValue()) { if (UndefMask.extractBits(HalfNumElems, HalfNumElems).isAllOnes()) {
EVT HalfVT = EVT HalfVT =
EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), HalfNumElems); EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), HalfNumElems);
SDValue HalfLD = SDValue HalfLD =
EltsFromConsecutiveLoads(HalfVT, Elts.drop_back(HalfNumElems), DL, EltsFromConsecutiveLoads(HalfVT, Elts.drop_back(HalfNumElems), DL,
DAG, Subtarget, IsAfterLegalize); DAG, Subtarget, IsAfterLegalize);
if (HalfLD) if (HalfLD)
return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT),
HalfLD, DAG.getIntPtrConstant(0, DL)); HalfLD, DAG.getIntPtrConstant(0, DL));
Show All 22 Lines if (TLI.isTypeLegal(VecVT)) {
if (LD) if (LD)
DAG.makeEquivalentMemoryOrdering(LD, ResNode); DAG.makeEquivalentMemoryOrdering(LD, ResNode);
return DAG.getBitcast(VT, ResNode); return DAG.getBitcast(VT, ResNode);
} }
} }
// BROADCAST - match the smallest possible repetition pattern, load that // BROADCAST - match the smallest possible repetition pattern, load that
// scalar/subvector element and then broadcast to the entire vector. // scalar/subvector element and then broadcast to the entire vector.
if (ZeroMask.isNullValue() && isPowerOf2_32(NumElems) && Subtarget.hasAVX() && if (ZeroMask.isZero() && isPowerOf2_32(NumElems) && Subtarget.hasAVX() &&
(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector())) { (VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector())) {
for (unsigned SubElems = 1; SubElems < NumElems; SubElems *= 2) { for (unsigned SubElems = 1; SubElems < NumElems; SubElems *= 2) {
unsigned RepeatSize = SubElems * BaseSizeInBits; unsigned RepeatSize = SubElems * BaseSizeInBits;
unsigned ScalarSize = std::min(RepeatSize, 64u); unsigned ScalarSize = std::min(RepeatSize, 64u);
if (!Subtarget.hasAVX2() && ScalarSize < 32) if (!Subtarget.hasAVX2() && ScalarSize < 32)
continue; continue;
// Don't attempt a 1:N subvector broadcast - it should be caught by // Don't attempt a 1:N subvector broadcast - it should be caught by
▲ Show 20 Lines • Show All 1,646 Lines • ▼ Show 20 Lines if (X86::isZeroNode(Elt)) {
ZeroMask.setBit(i); ZeroMask.setBit(i);
} else { } else {
NonZeroMask.setBit(i); NonZeroMask.setBit(i);
} }
} }
// All undef vector. Return an UNDEF. All zero vectors were handled above. // All undef vector. Return an UNDEF. All zero vectors were handled above.
if (NonZeroMask == 0) { if (NonZeroMask == 0) {
assert(UndefMask.isAllOnesValue() && "Fully undef mask expected"); assert(UndefMask.isAllOnes() && "Fully undef mask expected");
return DAG.getUNDEF(VT); return DAG.getUNDEF(VT);
} }
BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Op.getNode()); BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Op.getNode());
// If the upper elts of a ymm/zmm are undef/zero then we might be better off // If the upper elts of a ymm/zmm are undef/zero then we might be better off
// lowering to a smaller build vector and padding with undef/zero. // lowering to a smaller build vector and padding with undef/zero.
if ((VT.is256BitVector() || VT.is512BitVector()) && if ((VT.is256BitVector() || VT.is512BitVector()) &&
▲ Show 20 Lines • Show All 830 Lines • ▼ Show 20 Linesstatic bool createShuffleMaskFromVSELECT(SmallVectorImpl<int> &Mask,
Mask.resize(NumElts, SM_SentinelUndef); Mask.resize(NumElts, SM_SentinelUndef);
for (int i = 0; i != (int)NumElts; ++i) { for (int i = 0; i != (int)NumElts; ++i) {
Mask[i] = i; Mask[i] = i;
// Arbitrarily choose from the 2nd operand if the select condition element // Arbitrarily choose from the 2nd operand if the select condition element
// is undef. // is undef.
// TODO: Can we do better by matching patterns such as even/odd? // TODO: Can we do better by matching patterns such as even/odd?
if (UndefElts[i] || EltBits[i].isNullValue()) if (UndefElts[i] || EltBits[i].isZero())
Mask[i] += NumElts; Mask[i] += NumElts;
} }
return true; return true;
} }
// Check if the shuffle mask is suitable for the AVX vpunpcklwd or vpunpckhwd // Check if the shuffle mask is suitable for the AVX vpunpcklwd or vpunpckhwd
// instructions. // instructions.
▲ Show 20 Lines • Show All 352 Lines • ▼ Show 20 Linesstatic bool matchShuffleAsVTRUNC(MVT &SrcVT, MVT &DstVT, MVT VT,
for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) { for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) {
unsigned SrcEltBits = EltSizeInBits * Scale; unsigned SrcEltBits = EltSizeInBits * Scale;
if (SrcEltBits < 32 && !Subtarget.hasBWI()) if (SrcEltBits < 32 && !Subtarget.hasBWI())
continue; continue;
unsigned NumSrcElts = NumElts / Scale; unsigned NumSrcElts = NumElts / Scale;
if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale)) if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale))
continue; continue;
unsigned UpperElts = NumElts - NumSrcElts; unsigned UpperElts = NumElts - NumSrcElts;
if (!Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnesValue()) if (!Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
continue; continue;
SrcVT = MVT::getIntegerVT(EltSizeInBits * Scale); SrcVT = MVT::getIntegerVT(EltSizeInBits * Scale);
SrcVT = MVT::getVectorVT(SrcVT, NumSrcElts); SrcVT = MVT::getVectorVT(SrcVT, NumSrcElts);
DstVT = MVT::getIntegerVT(EltSizeInBits); DstVT = MVT::getIntegerVT(EltSizeInBits);
if ((NumSrcElts * EltSizeInBits) >= 128) { if ((NumSrcElts * EltSizeInBits) >= 128) {
// ISD::TRUNCATE // ISD::TRUNCATE
DstVT = MVT::getVectorVT(DstVT, NumSrcElts); DstVT = MVT::getVectorVT(DstVT, NumSrcElts);
} else { } else {
▲ Show 20 Lines • Show All 80 Lines • ▼ Show 20 Linesstatic SDValue lowerShuffleWithVPMOV(const SDLoc &DL, MVT VT, SDValue V1,
unsigned NumElts = VT.getVectorNumElements(); unsigned NumElts = VT.getVectorNumElements();
unsigned EltSizeInBits = VT.getScalarSizeInBits(); unsigned EltSizeInBits = VT.getScalarSizeInBits();
unsigned MaxScale = 64 / EltSizeInBits; unsigned MaxScale = 64 / EltSizeInBits;
for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) { for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) {
unsigned NumSrcElts = NumElts / Scale; unsigned NumSrcElts = NumElts / Scale;
unsigned UpperElts = NumElts - NumSrcElts; unsigned UpperElts = NumElts - NumSrcElts;
if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale) || if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale) ||
!Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnesValue()) !Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
continue; continue;
SDValue Src = V1; SDValue Src = V1;
if (!Src.hasOneUse()) if (!Src.hasOneUse())
return SDValue(); return SDValue();
Src = peekThroughOneUseBitcasts(Src); Src = peekThroughOneUseBitcasts(Src);
if (Src.getOpcode() != ISD::TRUNCATE || if (Src.getOpcode() != ISD::TRUNCATE ||
Show All 40 Lines for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) {
unsigned NumSrcElts = 2 * NumHalfSrcElts; unsigned NumSrcElts = 2 * NumHalfSrcElts;
if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale) || if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale) ||
isUndefInRange(Mask, NumHalfSrcElts, NumHalfSrcElts)) isUndefInRange(Mask, NumHalfSrcElts, NumHalfSrcElts))
continue; continue;
// The elements beyond the truncation must be undef/zero. // The elements beyond the truncation must be undef/zero.
unsigned UpperElts = NumElts - NumSrcElts; unsigned UpperElts = NumElts - NumSrcElts;
if (UpperElts > 0 && if (UpperElts > 0 &&
!Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnesValue()) !Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
continue; continue;
bool UndefUppers = bool UndefUppers =
UpperElts > 0 && isUndefInRange(Mask, NumSrcElts, UpperElts); UpperElts > 0 && isUndefInRange(Mask, NumSrcElts, UpperElts);
// As we're using both sources then we need to concat them together // As we're using both sources then we need to concat them together
// and truncate from the double-sized src. // and truncate from the double-sized src.
MVT ConcatVT = MVT::getVectorVT(VT.getScalarType(), NumElts * 2); MVT ConcatVT = MVT::getVectorVT(VT.getScalarType(), NumElts * 2);
SDValue Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2); SDValue Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2);
▲ Show 20 Lines • Show All 1,273 Lines • ▼ Show 20 Lines
// EXTRQ: Extract Len elements from lower half of source, starting at Idx. // EXTRQ: Extract Len elements from lower half of source, starting at Idx.
// Remainder of lower half result is zero and upper half is all undef. // Remainder of lower half result is zero and upper half is all undef.
static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2, static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2,
ArrayRef<int> Mask, uint64_t &BitLen, ArrayRef<int> Mask, uint64_t &BitLen,
uint64_t &BitIdx, const APInt &Zeroable) { uint64_t &BitIdx, const APInt &Zeroable) {
int Size = Mask.size(); int Size = Mask.size();
int HalfSize = Size / 2; int HalfSize = Size / 2;
assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size"); assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
assert(!Zeroable.isAllOnesValue() && "Fully zeroable shuffle mask"); assert(!Zeroable.isAllOnes() && "Fully zeroable shuffle mask");
// Upper half must be undefined. // Upper half must be undefined.
if (!isUndefUpperHalf(Mask)) if (!isUndefUpperHalf(Mask))
return false; return false;
// Determine the extraction length from the part of the // Determine the extraction length from the part of the
// lower half that isn't zeroable. // lower half that isn't zeroable.
int Len = HalfSize; int Len = HalfSize;
▲ Show 20 Lines • Show All 5,553 Lines • ▼ Show 20 Linesstatic SDValue lowerVECTOR_SHUFFLE(SDValue Op, const X86Subtarget &Subtarget,
// We actually see shuffles that are entirely re-arrangements of a set of // We actually see shuffles that are entirely re-arrangements of a set of
// zero inputs. This mostly happens while decomposing complex shuffles into // zero inputs. This mostly happens while decomposing complex shuffles into
// simple ones. Directly lower these as a buildvector of zeros. // simple ones. Directly lower these as a buildvector of zeros.
APInt KnownUndef, KnownZero; APInt KnownUndef, KnownZero;
computeZeroableShuffleElements(OrigMask, V1, V2, KnownUndef, KnownZero); computeZeroableShuffleElements(OrigMask, V1, V2, KnownUndef, KnownZero);
APInt Zeroable = KnownUndef | KnownZero; APInt Zeroable = KnownUndef | KnownZero;
if (Zeroable.isAllOnesValue()) if (Zeroable.isAllOnes())
return getZeroVector(VT, Subtarget, DAG, DL); return getZeroVector(VT, Subtarget, DAG, DL);
bool V2IsZero = !V2IsUndef && ISD::isBuildVectorAllZeros(V2.getNode()); bool V2IsZero = !V2IsUndef && ISD::isBuildVectorAllZeros(V2.getNode());
// Try to collapse shuffles into using a vector type with fewer elements but // Try to collapse shuffles into using a vector type with fewer elements but
// wider element types. We cap this to not form integers or floating point // wider element types. We cap this to not form integers or floating point
// elements wider than 64 bits. It does not seem beneficial to form i128 // elements wider than 64 bits. It does not seem beneficial to form i128
// integers to handle flipping the low and high halves of AVX 256-bit vectors. // integers to handle flipping the low and high halves of AVX 256-bit vectors.
▲ Show 20 Lines • Show All 3,907 Lines • ▼ Show 20 Linesstatic bool matchScalarReduction(SDValue Op, ISD::NodeType BinOp,
if (SrcMask) { if (SrcMask) {
// Collect the source partial masks. // Collect the source partial masks.
for (SDValue &SrcOp : SrcOps) for (SDValue &SrcOp : SrcOps)
SrcMask->push_back(SrcOpMap[SrcOp]); SrcMask->push_back(SrcOpMap[SrcOp]);
} else { } else {
// Quit if not all elements are used. // Quit if not all elements are used.
for (const auto &I : SrcOpMap) for (const auto &I : SrcOpMap)
if (!I.second.isAllOnesValue()) if (!I.second.isAllOnes())
return false; return false;
} }
return true; return true;
} }
// Helper function for comparing all bits of a vector against zero. // Helper function for comparing all bits of a vector against zero.
static SDValue LowerVectorAllZero(const SDLoc &DL, SDValue V, ISD::CondCode CC, static SDValue LowerVectorAllZero(const SDLoc &DL, SDValue V, ISD::CondCode CC,
const APInt &Mask, const APInt &Mask,
const X86Subtarget &Subtarget, const X86Subtarget &Subtarget,
SelectionDAG &DAG, X86::CondCode &X86CC) { SelectionDAG &DAG, X86::CondCode &X86CC) {
EVT VT = V.getValueType(); EVT VT = V.getValueType();
unsigned ScalarSize = VT.getScalarSizeInBits(); unsigned ScalarSize = VT.getScalarSizeInBits();
if (Mask.getBitWidth() != ScalarSize) { if (Mask.getBitWidth() != ScalarSize) {
assert(ScalarSize == 1 && "Element Mask vs Vector bitwidth mismatch"); assert(ScalarSize == 1 && "Element Mask vs Vector bitwidth mismatch");
return SDValue(); return SDValue();
} }
assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode"); assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode");
X86CC = (CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE); X86CC = (CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE);
auto MaskBits = [&](SDValue Src) { auto MaskBits = [&](SDValue Src) {
if (Mask.isAllOnesValue()) if (Mask.isAllOnes())
return Src; return Src;
EVT SrcVT = Src.getValueType(); EVT SrcVT = Src.getValueType();
SDValue MaskValue = DAG.getConstant(Mask, DL, SrcVT); SDValue MaskValue = DAG.getConstant(Mask, DL, SrcVT);
return DAG.getNode(ISD::AND, DL, SrcVT, Src, MaskValue); return DAG.getNode(ISD::AND, DL, SrcVT, Src, MaskValue);
}; };
// For sub-128-bit vector, cast to (legal) integer and compare with zero. // For sub-128-bit vector, cast to (legal) integer and compare with zero.
if (VT.getSizeInBits() < 128) { if (VT.getSizeInBits() < 128) {
Show All 21 Linesstatic SDValue LowerVectorAllZero(const SDLoc &DL, SDValue V, ISD::CondCode CC,
if (UsePTEST) { if (UsePTEST) {
MVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64; MVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
V = DAG.getBitcast(TestVT, MaskBits(V)); V = DAG.getBitcast(TestVT, MaskBits(V));
return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, V, V); return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, V, V);
} }
// Without PTEST, a masked v2i64 or-reduction is not faster than // Without PTEST, a masked v2i64 or-reduction is not faster than
// scalarization. // scalarization.
if (!Mask.isAllOnesValue() && VT.getScalarSizeInBits() > 32) if (!Mask.isAllOnes() && VT.getScalarSizeInBits() > 32)
return SDValue(); return SDValue();
V = DAG.getBitcast(MVT::v16i8, MaskBits(V)); V = DAG.getBitcast(MVT::v16i8, MaskBits(V));
V = DAG.getNode(X86ISD::PCMPEQ, DL, MVT::v16i8, V, V = DAG.getNode(X86ISD::PCMPEQ, DL, MVT::v16i8, V,
getZeroVector(MVT::v16i8, Subtarget, DAG, DL)); getZeroVector(MVT::v16i8, Subtarget, DAG, DL));
V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V); V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V, return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V,
DAG.getConstant(0xFFFF, DL, MVT::i32)); DAG.getConstant(0xFFFF, DL, MVT::i32));
} }
▲ Show 20 Lines • Show All 650 Lines • ▼ Show 20 Linesstatic SDValue incDecVectorConstant(SDValue V, SelectionDAG &DAG, bool IsInc) {
SDLoc DL(V); SDLoc DL(V);
for (unsigned i = 0; i < NumElts; ++i) { for (unsigned i = 0; i < NumElts; ++i) {
auto *Elt = dyn_cast<ConstantSDNode>(BV->getOperand(i)); auto *Elt = dyn_cast<ConstantSDNode>(BV->getOperand(i));
if (!Elt || Elt->isOpaque() || Elt->getSimpleValueType(0) != EltVT) if (!Elt || Elt->isOpaque() || Elt->getSimpleValueType(0) != EltVT)
return SDValue(); return SDValue();
// Avoid overflow/underflow. // Avoid overflow/underflow.
const APInt &EltC = Elt->getAPIntValue(); const APInt &EltC = Elt->getAPIntValue();
if ((IsInc && EltC.isMaxValue()) || (!IsInc && EltC.isNullValue())) if ((IsInc && EltC.isMaxValue()) || (!IsInc && EltC.isZero()))
return SDValue(); return SDValue();
NewVecC.push_back(DAG.getConstant(EltC + (IsInc ? 1 : -1), DL, EltVT)); NewVecC.push_back(DAG.getConstant(EltC + (IsInc ? 1 : -1), DL, EltVT));
} }
return DAG.getBuildVector(VT, DL, NewVecC); return DAG.getBuildVector(VT, DL, NewVecC);
} }
▲ Show 20 Lines • Show All 302 Lines • ▼ Show 20 Lines#endif
// +X != 0 --> +X >s 0 // +X != 0 --> +X >s 0
APInt ConstValue; APInt ConstValue;
if (Cond == ISD::SETNE && if (Cond == ISD::SETNE &&
ISD::isConstantSplatVector(Op1.getNode(), ConstValue)) { ISD::isConstantSplatVector(Op1.getNode(), ConstValue)) {
if (ConstValue.isMinSignedValue()) if (ConstValue.isMinSignedValue())
Cond = ISD::SETGT; Cond = ISD::SETGT;
else if (ConstValue.isMaxSignedValue()) else if (ConstValue.isMaxSignedValue())
Cond = ISD::SETLT; Cond = ISD::SETLT;
else if (ConstValue.isNullValue() && DAG.SignBitIsZero(Op0)) else if (ConstValue.isZero() && DAG.SignBitIsZero(Op0))
Cond = ISD::SETGT; Cond = ISD::SETGT;
} }
// If both operands are known non-negative, then an unsigned compare is the // If both operands are known non-negative, then an unsigned compare is the
// same as a signed compare and there's no need to flip signbits. // same as a signed compare and there's no need to flip signbits.
// TODO: We could check for more general simplifications here since we're // TODO: We could check for more general simplifications here since we're
// computing known bits. // computing known bits.
bool FlipSigns = ISD::isUnsignedIntSetCC(Cond) && bool FlipSigns = ISD::isUnsignedIntSetCC(Cond) &&
▲ Show 20 Lines • Show All 336 Lines • ▼ Show 20 Lines if (Op0.getSimpleValueType().isInteger()) {
// canonicalize to that CondCode. // canonicalize to that CondCode.
// NOTE: Only do this if incrementing the constant doesn't increase the bit // NOTE: Only do this if incrementing the constant doesn't increase the bit
// encoding size - so it must either already be a i8 or i32 immediate, or it // encoding size - so it must either already be a i8 or i32 immediate, or it
// shrinks down to that. We don't do this for any i64's to avoid additional // shrinks down to that. We don't do this for any i64's to avoid additional
// constant materializations. // constant materializations.
// TODO: Can we move this to TranslateX86CC to handle jumps/branches too? // TODO: Can we move this to TranslateX86CC to handle jumps/branches too?
if (auto *Op1C = dyn_cast<ConstantSDNode>(Op1)) { if (auto *Op1C = dyn_cast<ConstantSDNode>(Op1)) {
const APInt &Op1Val = Op1C->getAPIntValue(); const APInt &Op1Val = Op1C->getAPIntValue();
if (!Op1Val.isNullValue()) { if (!Op1Val.isZero()) {
// Ensure the constant+1 doesn't overflow. // Ensure the constant+1 doesn't overflow.
if ((CC == ISD::CondCode::SETGT && !Op1Val.isMaxSignedValue()) || if ((CC == ISD::CondCode::SETGT && !Op1Val.isMaxSignedValue()) ||
(CC == ISD::CondCode::SETUGT && !Op1Val.isMaxValue())) { (CC == ISD::CondCode::SETUGT && !Op1Val.isMaxValue())) {
APInt Op1ValPlusOne = Op1Val + 1; APInt Op1ValPlusOne = Op1Val + 1;
if (Op1ValPlusOne.isSignedIntN(32) && if (Op1ValPlusOne.isSignedIntN(32) &&
(!Op1Val.isSignedIntN(8) || Op1ValPlusOne.isSignedIntN(8))) { (!Op1Val.isSignedIntN(8) || Op1ValPlusOne.isSignedIntN(8))) {
Op1 = DAG.getConstant(Op1ValPlusOne, dl, Op0.getValueType()); Op1 = DAG.getConstant(Op1ValPlusOne, dl, Op0.getValueType());
CC = CC == ISD::CondCode::SETGT ? ISD::CondCode::SETGE CC = CC == ISD::CondCode::SETGT ? ISD::CondCode::SETGE
▲ Show 20 Lines • Show All 13,542 Lines • ▼ Show 20 Lines for (unsigned i = 0; i != NumMaskElts; ++i) {
if (!Bits) { if (!Bits) {
ZeroElts.setBit(i); ZeroElts.setBit(i);
continue; continue;
} }
ConstantElts.setBit(i); ConstantElts.setBit(i);
ConstantBitData[i] = Bits; ConstantBitData[i] = Bits;
} }
assert((UndefElts | ZeroElts | ConstantElts).isAllOnesValue()); assert((UndefElts | ZeroElts | ConstantElts).isAllOnes());
// Attempt to create a zero vector. // Attempt to create a zero vector.
if ((UndefElts | ZeroElts).isAllOnesValue()) if ((UndefElts | ZeroElts).isAllOnes())
return getZeroVector(Root.getSimpleValueType(), Subtarget, DAG, DL); return getZeroVector(Root.getSimpleValueType(), Subtarget, DAG, DL);
// Create the constant data. // Create the constant data.
MVT MaskSVT; MVT MaskSVT;
if (VT.isFloatingPoint() && (MaskSizeInBits == 32 || MaskSizeInBits == 64)) if (VT.isFloatingPoint() && (MaskSizeInBits == 32 || MaskSizeInBits == 64))
MaskSVT = MVT::getFloatingPointVT(MaskSizeInBits); MaskSVT = MVT::getFloatingPointVT(MaskSizeInBits);
else else
MaskSVT = MVT::getIntegerVT(MaskSizeInBits); MaskSVT = MVT::getIntegerVT(MaskSizeInBits);
▲ Show 20 Lines • Show All 118 Lines • ▼ Show 20 Linesstatic SDValue combineX86ShufflesRecursively(
SmallVector<SDValue, 16> Ops; SmallVector<SDValue, 16> Ops;
// We don't need to merge masks if the root is empty. // We don't need to merge masks if the root is empty.
bool EmptyRoot = (Depth == 0) && (RootMask.size() == 1); bool EmptyRoot = (Depth == 0) && (RootMask.size() == 1);
if (EmptyRoot) { if (EmptyRoot) {
// Only resolve zeros if it will remove an input, otherwise we might end // Only resolve zeros if it will remove an input, otherwise we might end
// up in an infinite loop. // up in an infinite loop.
bool ResolveKnownZeros = true; bool ResolveKnownZeros = true;
if (!OpZero.isNullValue()) { if (!OpZero.isZero()) {
APInt UsedInputs = APInt::getZero(OpInputs.size()); APInt UsedInputs = APInt::getZero(OpInputs.size());
for (int i = 0, e = OpMask.size(); i != e; ++i) { for (int i = 0, e = OpMask.size(); i != e; ++i) {
int M = OpMask[i]; int M = OpMask[i];
if (OpUndef[i] || OpZero[i] || isUndefOrZero(M)) if (OpUndef[i] || OpZero[i] || isUndefOrZero(M))
continue; continue;
UsedInputs.setBit(M / OpMask.size()); UsedInputs.setBit(M / OpMask.size());
if (UsedInputs.isAllOnesValue()) { if (UsedInputs.isAllOnes()) {
ResolveKnownZeros = false; ResolveKnownZeros = false;
break; break;
} }
} }
} }
resolveTargetShuffleFromZeroables(OpMask, OpUndef, OpZero, resolveTargetShuffleFromZeroables(OpMask, OpUndef, OpZero,
ResolveKnownZeros); ResolveKnownZeros);
▲ Show 20 Lines • Show All 1,670 Lines • ▼ Show 20 Lines
// Simplify variable target shuffle masks based on the demanded elements. // Simplify variable target shuffle masks based on the demanded elements.
// TODO: Handle DemandedBits in mask indices as well? // TODO: Handle DemandedBits in mask indices as well?
bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetShuffle( bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetShuffle(
SDValue Op, const APInt &DemandedElts, unsigned MaskIndex, SDValue Op, const APInt &DemandedElts, unsigned MaskIndex,
TargetLowering::TargetLoweringOpt &TLO, unsigned Depth) const { TargetLowering::TargetLoweringOpt &TLO, unsigned Depth) const {
// If we're demanding all elements don't bother trying to simplify the mask. // If we're demanding all elements don't bother trying to simplify the mask.
unsigned NumElts = DemandedElts.getBitWidth(); unsigned NumElts = DemandedElts.getBitWidth();
if (DemandedElts.isAllOnesValue()) if (DemandedElts.isAllOnes())
return false; return false;
SDValue Mask = Op.getOperand(MaskIndex); SDValue Mask = Op.getOperand(MaskIndex);
if (!Mask.hasOneUse()) if (!Mask.hasOneUse())
return false; return false;
// Attempt to generically simplify the variable shuffle mask. // Attempt to generically simplify the variable shuffle mask.
APInt MaskUndef, MaskZero; APInt MaskUndef, MaskZero;
▲ Show 20 Lines • Show All 79 Lines • ▼ Show 20 Lines case X86ISD::PSADBW: {
SDValue LHS = Op.getOperand(0); SDValue LHS = Op.getOperand(0);
SDValue RHS = Op.getOperand(1); SDValue RHS = Op.getOperand(1);
assert(VT.getScalarType() == MVT::i64 && assert(VT.getScalarType() == MVT::i64 &&
LHS.getValueType() == RHS.getValueType() && LHS.getValueType() == RHS.getValueType() &&
LHS.getValueType().getScalarType() == MVT::i8 && LHS.getValueType().getScalarType() == MVT::i8 &&
"Unexpected PSADBW types"); "Unexpected PSADBW types");
// Aggressively peek through ops to get at the demanded elts. // Aggressively peek through ops to get at the demanded elts.
if (!DemandedElts.isAllOnesValue()) { if (!DemandedElts.isAllOnes()) {
unsigned NumSrcElts = LHS.getValueType().getVectorNumElements(); unsigned NumSrcElts = LHS.getValueType().getVectorNumElements();
APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts); APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
SDValue NewLHS = SimplifyMultipleUseDemandedVectorElts( SDValue NewLHS = SimplifyMultipleUseDemandedVectorElts(
LHS, DemandedSrcElts, TLO.DAG, Depth + 1); LHS, DemandedSrcElts, TLO.DAG, Depth + 1);
SDValue NewRHS = SimplifyMultipleUseDemandedVectorElts( SDValue NewRHS = SimplifyMultipleUseDemandedVectorElts(
RHS, DemandedSrcElts, TLO.DAG, Depth + 1); RHS, DemandedSrcElts, TLO.DAG, Depth + 1);
if (NewLHS || NewRHS) { if (NewLHS || NewRHS) {
NewLHS = NewLHS ? NewLHS : LHS; NewLHS = NewLHS ? NewLHS : LHS;
Show All 34 Linesbool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
case X86ISD::VSRAI: { case X86ISD::VSRAI: {
SDValue Src = Op.getOperand(0); SDValue Src = Op.getOperand(0);
APInt SrcUndef; APInt SrcUndef;
if (SimplifyDemandedVectorElts(Src, DemandedElts, SrcUndef, KnownZero, TLO, if (SimplifyDemandedVectorElts(Src, DemandedElts, SrcUndef, KnownZero, TLO,
Depth + 1)) Depth + 1))
return true; return true;
// Aggressively peek through ops to get at the demanded elts. // Aggressively peek through ops to get at the demanded elts.
if (!DemandedElts.isAllOnesValue()) if (!DemandedElts.isAllOnes())
if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts( if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
Src, DemandedElts, TLO.DAG, Depth + 1)) Src, DemandedElts, TLO.DAG, Depth + 1))
return TLO.CombineTo( return TLO.CombineTo(
Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc, Op.getOperand(1))); Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc, Op.getOperand(1)));
break; break;
} }
case X86ISD::KSHIFTL: { case X86ISD::KSHIFTL: {
SDValue Src = Op.getOperand(0); SDValue Src = Op.getOperand(0);
▲ Show 20 Lines • Show All 100 Lines • ▼ Show 20 Lines case X86ISD::PACKUS: {
if (SimplifyDemandedVectorElts(N1, DemandedRHS, RHSUndef, RHSZero, TLO, if (SimplifyDemandedVectorElts(N1, DemandedRHS, RHSUndef, RHSZero, TLO,
Depth + 1)) Depth + 1))
return true; return true;
// TODO - pass on known zero/undef. // TODO - pass on known zero/undef.
// Aggressively peek through ops to get at the demanded elts. // Aggressively peek through ops to get at the demanded elts.
// TODO - we should do this for all target/faux shuffles ops. // TODO - we should do this for all target/faux shuffles ops.
if (!DemandedElts.isAllOnesValue()) { if (!DemandedElts.isAllOnes()) {
SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS, SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS,
TLO.DAG, Depth + 1); TLO.DAG, Depth + 1);
SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS, SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS,
TLO.DAG, Depth + 1); TLO.DAG, Depth + 1);
if (NewN0 || NewN1) { if (NewN0 || NewN1) {
NewN0 = NewN0 ? NewN0 : N0; NewN0 = NewN0 ? NewN0 : N0;
NewN1 = NewN1 ? NewN1 : N1; NewN1 = NewN1 ? NewN1 : N1;
return TLO.CombineTo(Op, return TLO.CombineTo(Op,
Show All 20 Lines case X86ISD::FHSUB: {
if (SimplifyDemandedVectorElts(N1, DemandedRHS, RHSUndef, RHSZero, TLO, if (SimplifyDemandedVectorElts(N1, DemandedRHS, RHSUndef, RHSZero, TLO,
Depth + 1)) Depth + 1))
return true; return true;
// TODO - pass on known zero/undef. // TODO - pass on known zero/undef.
// Aggressively peek through ops to get at the demanded elts. // Aggressively peek through ops to get at the demanded elts.
// TODO: Handle repeated operands. // TODO: Handle repeated operands.
if (N0 != N1 && !DemandedElts.isAllOnesValue()) { if (N0 != N1 && !DemandedElts.isAllOnes()) {
SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS, SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS,
TLO.DAG, Depth + 1); TLO.DAG, Depth + 1);
SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS, SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS,
TLO.DAG, Depth + 1); TLO.DAG, Depth + 1);
if (NewN0 || NewN1) { if (NewN0 || NewN1) {
NewN0 = NewN0 ? NewN0 : N0; NewN0 = NewN0 ? NewN0 : N0;
NewN1 = NewN1 ? NewN1 : N1; NewN1 = NewN1 ? NewN1 : N1;
return TLO.CombineTo(Op, return TLO.CombineTo(Op,
▲ Show 20 Lines • Show All 256 Lines • ▼ Show 20 Lines case X86ISD::FHSUB: {
return TLO.CombineTo(Op, Insert); return TLO.CombineTo(Op, Insert);
} }
} }
} }
// For broadcasts, unless we *only* demand the 0'th element, // For broadcasts, unless we *only* demand the 0'th element,
// stop attempts at simplification here, we aren't going to improve things, // stop attempts at simplification here, we aren't going to improve things,
// this is better than any potential shuffle. // this is better than any potential shuffle.
if (isTargetShuffleSplat(Op) && !DemandedElts.isOneValue()) if (isTargetShuffleSplat(Op) && !DemandedElts.isOne())
return false; return false;
// Get target/faux shuffle mask. // Get target/faux shuffle mask.
APInt OpUndef, OpZero; APInt OpUndef, OpZero;
SmallVector<int, 64> OpMask; SmallVector<int, 64> OpMask;
SmallVector<SDValue, 2> OpInputs; SmallVector<SDValue, 2> OpInputs;
if (!getTargetShuffleInputs(Op, DemandedElts, OpInputs, OpMask, OpUndef, if (!getTargetShuffleInputs(Op, DemandedElts, OpInputs, OpMask, OpUndef,
OpZero, TLO.DAG, Depth, false)) OpZero, TLO.DAG, Depth, false))
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Linesbool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
// If we don't demand all elements, then attempt to combine to a simpler // If we don't demand all elements, then attempt to combine to a simpler
// shuffle. // shuffle.
// We need to convert the depth to something combineX86ShufflesRecursively // We need to convert the depth to something combineX86ShufflesRecursively
// can handle - so pretend its Depth == 0 again, and reduce the max depth // can handle - so pretend its Depth == 0 again, and reduce the max depth
// to match. This prevents combineX86ShuffleChain from returning a // to match. This prevents combineX86ShuffleChain from returning a
// combined shuffle that's the same as the original root, causing an // combined shuffle that's the same as the original root, causing an
// infinite loop. // infinite loop.
if (!DemandedElts.isAllOnesValue()) { if (!DemandedElts.isAllOnes()) {
assert(Depth < X86::MaxShuffleCombineDepth && "Depth out of range"); assert(Depth < X86::MaxShuffleCombineDepth && "Depth out of range");
SmallVector<int, 64> DemandedMask(NumElts, SM_SentinelUndef); SmallVector<int, 64> DemandedMask(NumElts, SM_SentinelUndef);
for (int i = 0; i != NumElts; ++i) for (int i = 0; i != NumElts; ++i)
if (DemandedElts[i]) if (DemandedElts[i])
DemandedMask[i] = i; DemandedMask[i] = i;
SDValue NewShuffle = combineX86ShufflesRecursively( SDValue NewShuffle = combineX86ShufflesRecursively(
▲ Show 20 Lines • Show All 2,363 Lines • ▼ Show 20 Lines if (TrueVal.slt(FalseVal)) {
Cond = DAG.getNOT(DL, Cond, MVT::i1); Cond = DAG.getNOT(DL, Cond, MVT::i1);
std::swap(TrueC, FalseC); std::swap(TrueC, FalseC);
} }
// select Cond, TC, FC --> (zext(Cond) * (TC - FC)) + FC // select Cond, TC, FC --> (zext(Cond) * (TC - FC)) + FC
SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
// Multiply condition by the difference if non-one. // Multiply condition by the difference if non-one.
if (!AbsDiff.isOneValue()) if (!AbsDiff.isOne())
R = DAG.getNode(ISD::MUL, DL, VT, R, DAG.getConstant(AbsDiff, DL, VT)); R = DAG.getNode(ISD::MUL, DL, VT, R, DAG.getConstant(AbsDiff, DL, VT));
// Add the base if non-zero. // Add the base if non-zero.
if (!FalseC->isZero()) if (!FalseC->isZero())
R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0)); R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0));
return R; return R;
} }
▲ Show 20 Lines • Show All 741 Lines • ▼ Show 20 Lines if (Comparison == NegAddend) {
auto LockOp = lowerAtomicArithWithLOCK(AtomicSub, DAG, Subtarget); auto LockOp = lowerAtomicArithWithLOCK(AtomicSub, DAG, Subtarget);
DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0), DAG.getUNDEF(CmpVT)); DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0), DAG.getUNDEF(CmpVT));
DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1)); DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1));
return LockOp; return LockOp;
} }
// We can handle comparisons with zero in a number of cases by manipulating // We can handle comparisons with zero in a number of cases by manipulating
// the CC used. // the CC used.
if (!Comparison.isNullValue()) if (!Comparison.isZero())
return SDValue(); return SDValue();
if (CC == X86::COND_S && Addend == 1) if (CC == X86::COND_S && Addend == 1)
CC = X86::COND_LE; CC = X86::COND_LE;
else if (CC == X86::COND_NS && Addend == 1) else if (CC == X86::COND_NS && Addend == 1)
CC = X86::COND_G; CC = X86::COND_G;
else if (CC == X86::COND_G && Addend == -1) else if (CC == X86::COND_G && Addend == -1)
CC = X86::COND_GE; CC = X86::COND_GE;
▲ Show 20 Lines • Show All 389 Lines • ▼ Show 20 Linesstatic SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC,
SDValue Vec = CmpOp.getOperand(0); SDValue Vec = CmpOp.getOperand(0);
MVT VecVT = Vec.getSimpleValueType(); MVT VecVT = Vec.getSimpleValueType();
assert((VecVT.is128BitVector() || VecVT.is256BitVector()) && assert((VecVT.is128BitVector() || VecVT.is256BitVector()) &&
"Unexpected MOVMSK operand"); "Unexpected MOVMSK operand");
unsigned NumElts = VecVT.getVectorNumElements(); unsigned NumElts = VecVT.getVectorNumElements();
unsigned NumEltBits = VecVT.getScalarSizeInBits(); unsigned NumEltBits = VecVT.getScalarSizeInBits();
bool IsAnyOf = CmpOpcode == X86ISD::CMP && CmpVal.isNullValue(); bool IsAnyOf = CmpOpcode == X86ISD::CMP && CmpVal.isZero();
bool IsAllOf = CmpOpcode == X86ISD::SUB && NumElts <= CmpBits && bool IsAllOf = CmpOpcode == X86ISD::SUB && NumElts <= CmpBits &&
CmpVal.isMask(NumElts); CmpVal.isMask(NumElts);
if (!IsAnyOf && !IsAllOf) if (!IsAnyOf && !IsAllOf)
return SDValue(); return SDValue();
// See if we can peek through to a vector with a wider element type, if the // See if we can peek through to a vector with a wider element type, if the
// signbits extend down to all the sub-elements as well. // signbits extend down to all the sub-elements as well.
// Calling MOVMSK with the wider type, avoiding the bitcast, helps expose // Calling MOVMSK with the wider type, avoiding the bitcast, helps expose
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines if (NumElts <= CmpBits &&
ShuffleInputs.size() == 1 && !isAnyZeroOrUndef(ShuffleMask) && ShuffleInputs.size() == 1 && !isAnyZeroOrUndef(ShuffleMask) &&
ShuffleInputs[0].getValueSizeInBits() == VecVT.getSizeInBits()) { ShuffleInputs[0].getValueSizeInBits() == VecVT.getSizeInBits()) {
unsigned NumShuffleElts = ShuffleMask.size(); unsigned NumShuffleElts = ShuffleMask.size();
APInt DemandedElts = APInt::getZero(NumShuffleElts); APInt DemandedElts = APInt::getZero(NumShuffleElts);
for (int M : ShuffleMask) { for (int M : ShuffleMask) {
assert(0 <= M && M < (int)NumShuffleElts && "Bad unary shuffle index"); assert(0 <= M && M < (int)NumShuffleElts && "Bad unary shuffle index");
DemandedElts.setBit(M); DemandedElts.setBit(M);
} }
if (DemandedElts.isAllOnesValue()) { if (DemandedElts.isAllOnes()) {
SDLoc DL(EFLAGS); SDLoc DL(EFLAGS);
SDValue Result = DAG.getBitcast(VecVT, ShuffleInputs[0]); SDValue Result = DAG.getBitcast(VecVT, ShuffleInputs[0]);
Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result); Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
Result = Result =
DAG.getZExtOrTrunc(Result, DL, EFLAGS.getOperand(0).getValueType()); DAG.getZExtOrTrunc(Result, DL, EFLAGS.getOperand(0).getValueType());
return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result, return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
EFLAGS.getOperand(1)); EFLAGS.getOperand(1));
} }
▲ Show 20 Lines • Show All 2,119 Lines • ▼ Show 20 Lines if ((VT.getScalarSizeInBits() % 8) == 0 &&
// Check that the constant bitmask masks whole bytes. // Check that the constant bitmask masks whole bytes.
APInt UndefElts; APInt UndefElts;
SmallVector<APInt, 64> EltBits; SmallVector<APInt, 64> EltBits;
if (VT == SrcVecVT.getScalarType() && if (VT == SrcVecVT.getScalarType() &&
N->getOperand(0)->isOnlyUserOf(SrcVec.getNode()) && N->getOperand(0)->isOnlyUserOf(SrcVec.getNode()) &&
getTargetConstantBitsFromNode(BitMask, 8, UndefElts, EltBits) && getTargetConstantBitsFromNode(BitMask, 8, UndefElts, EltBits) &&
llvm::all_of(EltBits, [](const APInt &M) { llvm::all_of(EltBits, [](const APInt &M) {
return M.isNullValue() || M.isAllOnesValue(); return M.isZero() || M.isAllOnes();
})) { })) {
unsigned NumElts = SrcVecVT.getVectorNumElements(); unsigned NumElts = SrcVecVT.getVectorNumElements();
unsigned Scale = SrcVecVT.getScalarSizeInBits() / 8; unsigned Scale = SrcVecVT.getScalarSizeInBits() / 8;
unsigned Idx = N->getOperand(0).getConstantOperandVal(1); unsigned Idx = N->getOperand(0).getConstantOperandVal(1);
// Create a root shuffle mask from the byte mask and the extracted index. // Create a root shuffle mask from the byte mask and the extracted index.
SmallVector<int, 16> ShuffleMask(NumElts * Scale, SM_SentinelUndef); SmallVector<int, 16> ShuffleMask(NumElts * Scale, SM_SentinelUndef);
for (unsigned i = 0; i != Scale; ++i) { for (unsigned i = 0; i != Scale; ++i) {
if (UndefElts[i]) if (UndefElts[i])
continue; continue;
int VecIdx = Scale * Idx + i; int VecIdx = Scale * Idx + i;
ShuffleMask[VecIdx] = ShuffleMask[VecIdx] = EltBits[i].isZero() ? SM_SentinelZero : VecIdx;
EltBits[i].isNullValue() ? SM_SentinelZero : VecIdx;
} }
if (SDValue Shuffle = combineX86ShufflesRecursively( if (SDValue Shuffle = combineX86ShufflesRecursively(
{SrcVec}, 0, SrcVec, ShuffleMask, {}, /*Depth*/ 1, {SrcVec}, 0, SrcVec, ShuffleMask, {}, /*Depth*/ 1,
X86::MaxShuffleCombineDepth, X86::MaxShuffleCombineDepth,
/*HasVarMask*/ false, /*AllowVarCrossLaneMask*/ true, /*HasVarMask*/ false, /*AllowVarCrossLaneMask*/ true,
/*AllowVarPerLaneMask*/ true, DAG, Subtarget)) /*AllowVarPerLaneMask*/ true, DAG, Subtarget))
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), VT, Shuffle, return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), VT, Shuffle,
▲ Show 20 Lines • Show All 6,099 Lines • ▼ Show 20 Linesstatic SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG) {
SDLoc DL(N); SDLoc DL(N);
// If this is a scalar to vector to v1i1 from an AND with 1, bypass the and. // If this is a scalar to vector to v1i1 from an AND with 1, bypass the and.
// This occurs frequently in our masked scalar intrinsic code and our // This occurs frequently in our masked scalar intrinsic code and our
// floating point select lowering with AVX512. // floating point select lowering with AVX512.
// TODO: SimplifyDemandedBits instead? // TODO: SimplifyDemandedBits instead?
if (VT == MVT::v1i1 && Src.getOpcode() == ISD::AND && Src.hasOneUse()) if (VT == MVT::v1i1 && Src.getOpcode() == ISD::AND && Src.hasOneUse())
if (auto *C = dyn_cast<ConstantSDNode>(Src.getOperand(1))) if (auto *C = dyn_cast<ConstantSDNode>(Src.getOperand(1)))
if (C->getAPIntValue().isOneValue()) if (C->getAPIntValue().isOne())
return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1,
Src.getOperand(0)); Src.getOperand(0));
// Combine scalar_to_vector of an extract_vector_elt into an extract_subvec. // Combine scalar_to_vector of an extract_vector_elt into an extract_subvec.
if (VT == MVT::v1i1 && Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT && if (VT == MVT::v1i1 && Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
Src.hasOneUse() && Src.getOperand(0).getValueType().isVector() && Src.hasOneUse() && Src.getOperand(0).getValueType().isVector() &&
Src.getOperand(0).getValueType().getVectorElementType() == MVT::i1) Src.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
if (auto *C = dyn_cast<ConstantSDNode>(Src.getOperand(1))) if (auto *C = dyn_cast<ConstantSDNode>(Src.getOperand(1)))
▲ Show 20 Lines • Show All 1,889 Lines • Show Last 20 Lines

llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp

Show First 20 Lines • Show All 233 Lines • ▼ Show 20 Lines if (IsImm) {
unsigned NumAmtElts = cast<FixedVectorType>(AmtVT)->getNumElements(); unsigned NumAmtElts = cast<FixedVectorType>(AmtVT)->getNumElements();
APInt DemandedLower = APInt::getOneBitSet(NumAmtElts, 0); APInt DemandedLower = APInt::getOneBitSet(NumAmtElts, 0);
APInt DemandedUpper = APInt::getBitsSet(NumAmtElts, 1, NumAmtElts / 2); APInt DemandedUpper = APInt::getBitsSet(NumAmtElts, 1, NumAmtElts / 2);
KnownBits KnownLowerBits = llvm::computeKnownBits( KnownBits KnownLowerBits = llvm::computeKnownBits(
Amt, DemandedLower, II.getModule()->getDataLayout()); Amt, DemandedLower, II.getModule()->getDataLayout());
KnownBits KnownUpperBits = llvm::computeKnownBits( KnownBits KnownUpperBits = llvm::computeKnownBits(
Amt, DemandedUpper, II.getModule()->getDataLayout()); Amt, DemandedUpper, II.getModule()->getDataLayout());
if (KnownLowerBits.getMaxValue().ult(BitWidth) && if (KnownLowerBits.getMaxValue().ult(BitWidth) &&
(DemandedUpper.isNullValue() || KnownUpperBits.isZero())) { (DemandedUpper.isZero() || KnownUpperBits.isZero())) {
SmallVector<int, 16> ZeroSplat(VWidth, 0); SmallVector<int, 16> ZeroSplat(VWidth, 0);
Amt = Builder.CreateShuffleVector(Amt, ZeroSplat); Amt = Builder.CreateShuffleVector(Amt, ZeroSplat);
return (LogicalShift ? (ShiftLeft ? Builder.CreateShl(Vec, Amt) return (LogicalShift ? (ShiftLeft ? Builder.CreateShl(Vec, Amt)
: Builder.CreateLShr(Vec, Amt)) : Builder.CreateLShr(Vec, Amt))
: Builder.CreateAShr(Vec, Amt)); : Builder.CreateAShr(Vec, Amt));
} }
} }
Show All 13 Linesstatic Value *simplifyX86immShift(const IntrinsicInst &II,
for (unsigned i = 0, NumSubElts = 64 / BitWidth; i != NumSubElts; ++i) { for (unsigned i = 0, NumSubElts = 64 / BitWidth; i != NumSubElts; ++i) {
unsigned SubEltIdx = (NumSubElts - 1) - i; unsigned SubEltIdx = (NumSubElts - 1) - i;
auto *SubElt = cast<ConstantInt>(CDV->getElementAsConstant(SubEltIdx)); auto *SubElt = cast<ConstantInt>(CDV->getElementAsConstant(SubEltIdx));
Count <<= BitWidth; Count <<= BitWidth;
Count |= SubElt->getValue().zextOrTrunc(64); Count |= SubElt->getValue().zextOrTrunc(64);
} }
// If shift-by-zero then just return the original value. // If shift-by-zero then just return the original value.
if (Count.isNullValue()) if (Count.isZero())
return Vec; return Vec;
// Handle cases when Shift >= BitWidth. // Handle cases when Shift >= BitWidth.
if (Count.uge(BitWidth)) { if (Count.uge(BitWidth)) {
// If LogicalShift - just return zero. // If LogicalShift - just return zero.
if (LogicalShift) if (LogicalShift)
return ConstantAggregateZero::get(VT); return ConstantAggregateZero::get(VT);
▲ Show 20 Lines • Show All 1,478 Lines • ▼ Show 20 Lines if (II.getIntrinsicID() == Intrinsic::x86_mmx_pmovmskb) {
auto *ArgType = cast<FixedVectorType>(II.getArgOperand(0)->getType()); auto *ArgType = cast<FixedVectorType>(II.getArgOperand(0)->getType());
ArgWidth = ArgType->getNumElements(); ArgWidth = ArgType->getNumElements();
} }
// If we don't need any of low bits then return zero, // If we don't need any of low bits then return zero,
// we know that DemandedMask is non-zero already. // we know that DemandedMask is non-zero already.
APInt DemandedElts = DemandedMask.zextOrTrunc(ArgWidth); APInt DemandedElts = DemandedMask.zextOrTrunc(ArgWidth);
Type *VTy = II.getType(); Type *VTy = II.getType();
if (DemandedElts.isNullValue()) { if (DemandedElts.isZero()) {
return ConstantInt::getNullValue(VTy); return ConstantInt::getNullValue(VTy);
} }
// We know that the upper bits are set to zero. // We know that the upper bits are set to zero.
Known.Zero.setBitsFrom(ArgWidth); Known.Zero.setBitsFrom(ArgWidth);
KnownBitsComputed = true; KnownBitsComputed = true;
break; break;
} }
▲ Show 20 Lines • Show All 241 Lines • Show Last 20 Lines

llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp

Show First 20 Lines • Show All 94 Lines • ▼ Show 20 Linesstatic bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
// Now extract the undef/constant bit data into the raw shuffle masks. // Now extract the undef/constant bit data into the raw shuffle masks.
for (unsigned i = 0; i != NumMaskElts; ++i) { for (unsigned i = 0; i != NumMaskElts; ++i) {
unsigned BitOffset = i * MaskEltSizeInBits; unsigned BitOffset = i * MaskEltSizeInBits;
APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset); APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);
// Only treat the element as UNDEF if all bits are UNDEF, otherwise // Only treat the element as UNDEF if all bits are UNDEF, otherwise
// treat it as zero. // treat it as zero.
if (EltUndef.isAllOnesValue()) { if (EltUndef.isAllOnes()) {
UndefElts.setBit(i); UndefElts.setBit(i);
RawMask[i] = 0; RawMask[i] = 0;
continue; continue;
} }
APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset); APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);
RawMask[i] = EltBits.getZExtValue(); RawMask[i] = EltBits.getZExtValue();
} }
▲ Show 20 Lines • Show All 185 Lines • Show Last 20 Lines

llvm/lib/Transforms/IPO/AttributorAttributes.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 8,723 Lines • ▼ Show 20 Lines default:
return LHS; return LHS;
case Instruction::Add: case Instruction::Add:
return LHS + RHS; return LHS + RHS;
case Instruction::Sub: case Instruction::Sub:
return LHS - RHS; return LHS - RHS;
case Instruction::Mul: case Instruction::Mul:
return LHS * RHS; return LHS * RHS;
case Instruction::UDiv: case Instruction::UDiv:
if (RHS.isNullValue()) { if (RHS.isZero()) {
SkipOperation = true; SkipOperation = true;
return LHS; return LHS;
} }
return LHS.udiv(RHS); return LHS.udiv(RHS);
case Instruction::SDiv: case Instruction::SDiv:
if (RHS.isNullValue()) { if (RHS.isZero()) {
SkipOperation = true; SkipOperation = true;
return LHS; return LHS;
} }
return LHS.sdiv(RHS); return LHS.sdiv(RHS);
case Instruction::URem: case Instruction::URem:
if (RHS.isNullValue()) { if (RHS.isZero()) {
SkipOperation = true; SkipOperation = true;
return LHS; return LHS;
} }
return LHS.urem(RHS); return LHS.urem(RHS);
case Instruction::SRem: case Instruction::SRem:
if (RHS.isNullValue()) { if (RHS.isZero()) {
SkipOperation = true; SkipOperation = true;
return LHS; return LHS;
} }
return LHS.srem(RHS); return LHS.srem(RHS);
case Instruction::Shl: case Instruction::Shl:
return LHS.shl(RHS); return LHS.shl(RHS);
case Instruction::LShr: case Instruction::LShr:
return LHS.lshr(RHS); return LHS.lshr(RHS);
▲ Show 20 Lines • Show All 1,082 Lines • Show Last 20 Lines

llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp

Show First 20 Lines • Show All 933 Lines • ▼ Show 20 Lines if (match(Op0, m_Xor(m_Value(X), m_APInt(C2)))) {
// (X ^ signmask) + C --> (X + (signmask ^ C)) // (X ^ signmask) + C --> (X + (signmask ^ C))
if (C2->isSignMask()) if (C2->isSignMask())
return BinaryOperator::CreateAdd(X, ConstantInt::get(Ty, *C2 ^ *C)); return BinaryOperator::CreateAdd(X, ConstantInt::get(Ty, *C2 ^ *C));
// If X has no high-bits set above an xor mask: // If X has no high-bits set above an xor mask:
// add (xor X, LowMaskC), C --> sub (LowMaskC + C), X // add (xor X, LowMaskC), C --> sub (LowMaskC + C), X
if (C2->isMask()) { if (C2->isMask()) {
KnownBits LHSKnown = computeKnownBits(X, 0, &Add); KnownBits LHSKnown = computeKnownBits(X, 0, &Add);
if ((*C2 | LHSKnown.Zero).isAllOnesValue()) if ((*C2 | LHSKnown.Zero).isAllOnes())
return BinaryOperator::CreateSub(ConstantInt::get(Ty, *C2 + *C), X); return BinaryOperator::CreateSub(ConstantInt::get(Ty, *C2 + *C), X);
} }
// Look for a math+logic pattern that corresponds to sext-in-register of a // Look for a math+logic pattern that corresponds to sext-in-register of a
// value with cleared high bits. Convert that into a pair of shifts: // value with cleared high bits. Convert that into a pair of shifts:
// add (xor X, 0x80), 0xF..F80 --> (X << ShAmtC) >>s ShAmtC // add (xor X, 0x80), 0xF..F80 --> (X << ShAmtC) >>s ShAmtC
// add (xor X, 0xF..F80), 0x80 --> (X << ShAmtC) >>s ShAmtC // add (xor X, 0xF..F80), 0x80 --> (X << ShAmtC) >>s ShAmtC
if (Op0->hasOneUse() && *C2 == -(*C)) { if (Op0->hasOneUse() && *C2 == -(*C)) {
unsigned BitWidth = Ty->getScalarSizeInBits(); unsigned BitWidth = Ty->getScalarSizeInBits();
unsigned ShAmt = 0; unsigned ShAmt = 0;
if (C->isPowerOf2()) if (C->isPowerOf2())
ShAmt = BitWidth - C->logBase2() - 1; ShAmt = BitWidth - C->logBase2() - 1;
else if (C2->isPowerOf2()) else if (C2->isPowerOf2())
ShAmt = BitWidth - C2->logBase2() - 1; ShAmt = BitWidth - C2->logBase2() - 1;
if (ShAmt && MaskedValueIsZero(X, APInt::getHighBitsSet(BitWidth, ShAmt), if (ShAmt && MaskedValueIsZero(X, APInt::getHighBitsSet(BitWidth, ShAmt),
0, &Add)) { 0, &Add)) {
Constant *ShAmtC = ConstantInt::get(Ty, ShAmt); Constant *ShAmtC = ConstantInt::get(Ty, ShAmt);
Value *NewShl = Builder.CreateShl(X, ShAmtC, "sext"); Value *NewShl = Builder.CreateShl(X, ShAmtC, "sext");
return BinaryOperator::CreateAShr(NewShl, ShAmtC); return BinaryOperator::CreateAShr(NewShl, ShAmtC);
} }
} }
} }
if (C->isOneValue() && Op0->hasOneUse()) { if (C->isOne() && Op0->hasOneUse()) {
// add (sext i1 X), 1 --> zext (not X) // add (sext i1 X), 1 --> zext (not X)
// TODO: The smallest IR representation is (select X, 0, 1), and that would // TODO: The smallest IR representation is (select X, 0, 1), and that would
// not require the one-use check. But we need to remove a transform in // not require the one-use check. But we need to remove a transform in
// visitSelect and make sure that IR value tracking for select is equal or // visitSelect and make sure that IR value tracking for select is equal or
// better than for these ops. // better than for these ops.
if (match(Op0, m_SExt(m_Value(X))) && if (match(Op0, m_SExt(m_Value(X))) &&
X->getType()->getScalarSizeInBits() == 1) X->getType()->getScalarSizeInBits() == 1)
return new ZExtInst(Builder.CreateNot(X), Ty); return new ZExtInst(Builder.CreateNot(X), Ty);
▲ Show 20 Lines • Show All 930 Lines • ▼ Show 20 Lines if (match(Op1, m_Sub(m_ImmConstant(C2), m_Value(X))))
return BinaryOperator::CreateAdd(X, ConstantExpr::getSub(C, C2)); return BinaryOperator::CreateAdd(X, ConstantExpr::getSub(C, C2));
} }
const APInt *Op0C; const APInt *Op0C;
if (match(Op0, m_APInt(Op0C)) && Op0C->isMask()) { if (match(Op0, m_APInt(Op0C)) && Op0C->isMask()) {
// Turn this into a xor if LHS is 2^n-1 and the remaining bits are known // Turn this into a xor if LHS is 2^n-1 and the remaining bits are known
// zero. // zero.
KnownBits RHSKnown = computeKnownBits(Op1, 0, &I); KnownBits RHSKnown = computeKnownBits(Op1, 0, &I);
if ((*Op0C | RHSKnown.Zero).isAllOnesValue()) if ((*Op0C | RHSKnown.Zero).isAllOnes())
return BinaryOperator::CreateXor(Op1, Op0); return BinaryOperator::CreateXor(Op1, Op0);
} }
{ {
Value *Y; Value *Y;
// X-(X+Y) == -Y X-(Y+X) == -Y // X-(X+Y) == -Y X-(Y+X) == -Y
if (match(Op1, m_c_Add(m_Specific(Op0), m_Value(Y)))) if (match(Op1, m_c_Add(m_Specific(Op0), m_Value(Y))))
return BinaryOperator::CreateNeg(Y); return BinaryOperator::CreateNeg(Y);
▲ Show 20 Lines • Show All 227 Lines • ▼ Show 20 LinesInstruction *InstCombinerImpl::visitSub(BinaryOperator &I) {
// of the sum: // of the sum:
// sub (X + AddC), (X & AndC) --> and (X + AddC), ~AndC // sub (X + AddC), (X & AndC) --> and (X + AddC), ~AndC
const APInt *AddC, *AndC; const APInt *AddC, *AndC;
if (match(Op0, m_Add(m_Value(X), m_APInt(AddC))) && if (match(Op0, m_Add(m_Value(X), m_APInt(AddC))) &&
match(Op1, m_And(m_Specific(X), m_APInt(AndC)))) { match(Op1, m_And(m_Specific(X), m_APInt(AndC)))) {
unsigned BitWidth = Ty->getScalarSizeInBits(); unsigned BitWidth = Ty->getScalarSizeInBits();
unsigned Cttz = AddC->countTrailingZeros(); unsigned Cttz = AddC->countTrailingZeros();
APInt HighMask(APInt::getHighBitsSet(BitWidth, BitWidth - Cttz)); APInt HighMask(APInt::getHighBitsSet(BitWidth, BitWidth - Cttz));
if ((HighMask & *AndC).isNullValue()) if ((HighMask & *AndC).isZero())
return BinaryOperator::CreateAnd(Op0, ConstantInt::get(Ty, ~(*AndC))); return BinaryOperator::CreateAnd(Op0, ConstantInt::get(Ty, ~(*AndC)));
} }
if (Instruction *V = if (Instruction *V =
canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract(I)) canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract(I))
return V; return V;
// X - usub.sat(X, Y) => umin(X, Y) // X - usub.sat(X, Y) => umin(X, Y)
▲ Show 20 Lines • Show All 285 Lines • Show Last 20 Lines

llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp

Show First 20 Lines • Show All 773 Lines • ▼ Show 20 Lines if (Xor.isPowerOf2()) {
// We choose an 'or' with a Pow2 constant rather than the inverse mask with // We choose an 'or' with a Pow2 constant rather than the inverse mask with
// 'and' because that may lead to smaller codegen from a smaller constant. // 'and' because that may lead to smaller codegen from a smaller constant.
Value *Or = Builder.CreateOr(X, ConstantInt::get(X->getType(), Xor)); Value *Or = Builder.CreateOr(X, ConstantInt::get(X->getType(), Xor));
return Builder.CreateICmp(Pred, Or, ConstantInt::get(X->getType(), *C2)); return Builder.CreateICmp(Pred, Or, ConstantInt::get(X->getType(), *C2));
} }
// Special case: get the ordering right when the values wrap around zero. // Special case: get the ordering right when the values wrap around zero.
// Ie, we assumed the constants were unsigned when swapping earlier. // Ie, we assumed the constants were unsigned when swapping earlier.
if (C1->isNullValue() && C2->isAllOnesValue()) if (C1->isZero() && C2->isAllOnes())
std::swap(C1, C2); std::swap(C1, C2);
if (*C1 == *C2 - 1) { if (*C1 == *C2 - 1) {
// (X == 13 || X == 14) --> X - 13 <=u 1 // (X == 13 || X == 14) --> X - 13 <=u 1
// (X != 13 && X != 14) --> X - 13 >u 1 // (X != 13 && X != 14) --> X - 13 >u 1
// An 'add' is the canonical IR form, so favor that over a 'sub'. // An 'add' is the canonical IR form, so favor that over a 'sub'.
Value *Add = Builder.CreateAdd(X, ConstantInt::get(X->getType(), -(*C1))); Value *Add = Builder.CreateAdd(X, ConstantInt::get(X->getType(), -(*C1)));
auto NewPred = JoinedByAnd ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_ULE; auto NewPred = JoinedByAnd ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_ULE;
▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Linesstatic Value *foldSignedTruncationCheck(ICmpInst *ICmp0, ICmpInst *ICmp1,
}; };
// And the other icmp needs to be decomposable into a bit test. // And the other icmp needs to be decomposable into a bit test.
Value *X0; Value *X0;
APInt UnsetBitsMask; APInt UnsetBitsMask;
if (!tryToDecompose(OtherICmp, X0, UnsetBitsMask)) if (!tryToDecompose(OtherICmp, X0, UnsetBitsMask))
return nullptr; return nullptr;
assert(!UnsetBitsMask.isNullValue() && "empty mask makes no sense."); assert(!UnsetBitsMask.isZero() && "empty mask makes no sense.");
// Are they working on the same value? // Are they working on the same value?
Value *X; Value *X;
if (X1 == X0) { if (X1 == X0) {
// Ok as is. // Ok as is.
X = X1; X = X1;
} else if (match(X0, m_Trunc(m_Specific(X1)))) { } else if (match(X0, m_Trunc(m_Specific(X1)))) {
UnsetBitsMask = UnsetBitsMask.zext(X1->getType()->getScalarSizeInBits()); UnsetBitsMask = UnsetBitsMask.zext(X1->getType()->getScalarSizeInBits());
▲ Show 20 Lines • Show All 370 Lines • ▼ Show 20 Lines if (PredL == ICmpInst::ICMP_EQ && PredL == PredR && LHS->hasOneUse() &&
} }
if (SmallC && BigC) { if (SmallC && BigC) {
unsigned BigBitSize = BigC->getType()->getBitWidth(); unsigned BigBitSize = BigC->getType()->getBitWidth();
unsigned SmallBitSize = SmallC->getType()->getBitWidth(); unsigned SmallBitSize = SmallC->getType()->getBitWidth();
// Check that the low bits are zero. // Check that the low bits are zero.
APInt Low = APInt::getLowBitsSet(BigBitSize, SmallBitSize); APInt Low = APInt::getLowBitsSet(BigBitSize, SmallBitSize);
if ((Low & AndC->getValue()).isNullValue() && if ((Low & AndC->getValue()).isZero() &&
(Low & BigC->getValue()).isNullValue()) { (Low & BigC->getValue()).isZero()) {
Value *NewAnd = Builder.CreateAnd(V, Low | AndC->getValue()); Value *NewAnd = Builder.CreateAnd(V, Low | AndC->getValue());
APInt N = SmallC->getValue().zext(BigBitSize) | BigC->getValue(); APInt N = SmallC->getValue().zext(BigBitSize) | BigC->getValue();
Value *NewVal = ConstantInt::get(AndC->getType()->getContext(), N); Value *NewVal = ConstantInt::get(AndC->getType()->getContext(), N);
return Builder.CreateICmp(PredL, NewAnd, NewVal); return Builder.CreateICmp(PredL, NewAnd, NewVal);
} }
} }
} }
▲ Show 20 Lines • Show All 555 Lines • ▼ Show 20 Lines if (match(Op1, m_APInt(C))) {
} }
const APInt *AddC; const APInt *AddC;
if (match(Op0, m_Add(m_Value(X), m_APInt(AddC)))) { if (match(Op0, m_Add(m_Value(X), m_APInt(AddC)))) {
// If we add zeros to every bit below a mask, the add has no effect: // If we add zeros to every bit below a mask, the add has no effect:
// (X + AddC) & LowMaskC --> X & LowMaskC // (X + AddC) & LowMaskC --> X & LowMaskC
unsigned Ctlz = C->countLeadingZeros(); unsigned Ctlz = C->countLeadingZeros();
APInt LowMask(APInt::getLowBitsSet(Width, Width - Ctlz)); APInt LowMask(APInt::getLowBitsSet(Width, Width - Ctlz));
if ((*AddC & LowMask).isNullValue()) if ((*AddC & LowMask).isZero())
return BinaryOperator::CreateAnd(X, Op1); return BinaryOperator::CreateAnd(X, Op1);
// If we are masking the result of the add down to exactly one bit and // If we are masking the result of the add down to exactly one bit and
// the constant we are adding has no bits set below that bit, then the // the constant we are adding has no bits set below that bit, then the
// add is flipping a single bit. Example: // add is flipping a single bit. Example:
// (X + 4) & 4 --> (X & 4) ^ 4 // (X + 4) & 4 --> (X & 4) ^ 4
if (Op0->hasOneUse() && C->isPowerOf2() && (*AddC & (*C - 1)) == 0) { if (Op0->hasOneUse() && C->isPowerOf2() && (*AddC & (*C - 1)) == 0) {
assert((*C & *AddC) != 0 && "Expected common bit"); assert((*C & *AddC) != 0 && "Expected common bit");
▲ Show 20 Lines • Show All 777 Lines • ▼ Show 20 Lines if (Instruction *Funnel = matchFunnelShift(I, *this))
return Funnel; return Funnel;
if (Instruction *Concat = matchOrConcat(I, Builder)) if (Instruction *Concat = matchOrConcat(I, Builder))
return replaceInstUsesWith(I, Concat); return replaceInstUsesWith(I, Concat);
Value *X, *Y; Value *X, *Y;
const APInt *CV; const APInt *CV;
if (match(&I, m_c_Or(m_OneUse(m_Xor(m_Value(X), m_APInt(CV))), m_Value(Y))) && if (match(&I, m_c_Or(m_OneUse(m_Xor(m_Value(X), m_APInt(CV))), m_Value(Y))) &&
!CV->isAllOnesValue() && MaskedValueIsZero(Y, *CV, 0, &I)) { !CV->isAllOnes() && MaskedValueIsZero(Y, *CV, 0, &I)) {
// (X ^ C) | Y -> (X | Y) ^ C iff Y & C == 0 // (X ^ C) | Y -> (X | Y) ^ C iff Y & C == 0
// The check for a 'not' op is for efficiency (if Y is known zero --> ~X). // The check for a 'not' op is for efficiency (if Y is known zero --> ~X).
Value *Or = Builder.CreateOr(X, Y); Value *Or = Builder.CreateOr(X, Y);
return BinaryOperator::CreateXor(Or, ConstantInt::get(I.getType(), *CV)); return BinaryOperator::CreateXor(Or, ConstantInt::get(I.getType(), *CV));
} }
// (A & C)|(B & D) // (A & C)|(B & D)
Value *A, *B, *C, *D; Value *A, *B, *C, *D;
if (match(Op0, m_And(m_Value(A), m_Value(C))) && if (match(Op0, m_And(m_Value(A), m_Value(C))) &&
match(Op1, m_And(m_Value(B), m_Value(D)))) { match(Op1, m_And(m_Value(B), m_Value(D)))) {
// (A & C1)|(B & C2) // (A & C1)|(B & C2)
ConstantInt *C1, *C2; ConstantInt *C1, *C2;
if (match(C, m_ConstantInt(C1)) && match(D, m_ConstantInt(C2))) { if (match(C, m_ConstantInt(C1)) && match(D, m_ConstantInt(C2))) {
Value *V1 = nullptr, *V2 = nullptr; Value *V1 = nullptr, *V2 = nullptr;
if ((C1->getValue() & C2->getValue()).isNullValue()) { if ((C1->getValue() & C2->getValue()).isZero()) {
// ((V | N) & C1) | (V & C2) --> (V|N) & (C1|C2) // ((V | N) & C1) | (V & C2) --> (V|N) & (C1|C2)
// iff (C1&C2) == 0 and (N&~C1) == 0 // iff (C1&C2) == 0 and (N&~C1) == 0
if (match(A, m_Or(m_Value(V1), m_Value(V2))) && if (match(A, m_Or(m_Value(V1), m_Value(V2))) &&
((V1 == B && ((V1 == B &&
MaskedValueIsZero(V2, ~C1->getValue(), 0, &I)) || // (V|N) MaskedValueIsZero(V2, ~C1->getValue(), 0, &I)) || // (V|N)
(V2 == B && (V2 == B &&
MaskedValueIsZero(V1, ~C1->getValue(), 0, &I)))) // (N|V) MaskedValueIsZero(V1, ~C1->getValue(), 0, &I)))) // (N|V)
return BinaryOperator::CreateAnd(A, return BinaryOperator::CreateAnd(A,
Builder.getInt(C1->getValue()|C2->getValue())); Builder.getInt(C1->getValue()|C2->getValue()));
// Or commutes, try both ways. // Or commutes, try both ways.
if (match(B, m_Or(m_Value(V1), m_Value(V2))) && if (match(B, m_Or(m_Value(V1), m_Value(V2))) &&
((V1 == A && ((V1 == A &&
MaskedValueIsZero(V2, ~C2->getValue(), 0, &I)) || // (V|N) MaskedValueIsZero(V2, ~C2->getValue(), 0, &I)) || // (V|N)
(V2 == A && (V2 == A &&
MaskedValueIsZero(V1, ~C2->getValue(), 0, &I)))) // (N|V) MaskedValueIsZero(V1, ~C2->getValue(), 0, &I)))) // (N|V)
return BinaryOperator::CreateAnd(B, return BinaryOperator::CreateAnd(B,
Builder.getInt(C1->getValue()|C2->getValue())); Builder.getInt(C1->getValue()|C2->getValue()));
// ((V|C3)&C1) | ((V|C4)&C2) --> (V|C3|C4)&(C1|C2) // ((V|C3)&C1) | ((V|C4)&C2) --> (V|C3|C4)&(C1|C2)
// iff (C1&C2) == 0 and (C3&~C1) == 0 and (C4&~C2) == 0. // iff (C1&C2) == 0 and (C3&~C1) == 0 and (C4&~C2) == 0.
ConstantInt *C3 = nullptr, *C4 = nullptr; ConstantInt *C3 = nullptr, *C4 = nullptr;
if (match(A, m_Or(m_Value(V1), m_ConstantInt(C3))) && if (match(A, m_Or(m_Value(V1), m_ConstantInt(C3))) &&
(C3->getValue() & ~C1->getValue()).isNullValue() && (C3->getValue() & ~C1->getValue()).isZero() &&
match(B, m_Or(m_Specific(V1), m_ConstantInt(C4))) && match(B, m_Or(m_Specific(V1), m_ConstantInt(C4))) &&
(C4->getValue() & ~C2->getValue()).isNullValue()) { (C4->getValue() & ~C2->getValue()).isZero()) {
V2 = Builder.CreateOr(V1, ConstantExpr::getOr(C3, C4), "bitfield"); V2 = Builder.CreateOr(V1, ConstantExpr::getOr(C3, C4), "bitfield");
return BinaryOperator::CreateAnd(V2, return BinaryOperator::CreateAnd(V2,
Builder.getInt(C1->getValue()|C2->getValue())); Builder.getInt(C1->getValue()|C2->getValue()));
} }
} }
if (C1->getValue() == ~C2->getValue()) { if (C1->getValue() == ~C2->getValue()) {
Value *X; Value *X;
▲ Show 20 Lines • Show All 953 Lines • Show Last 20 Lines

llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp

Show First 20 Lines • Show All 507 Lines • ▼ Show 20 Linesstatic Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC) {
if (PossibleZeros == DefiniteZeros) { if (PossibleZeros == DefiniteZeros) {
auto *C = ConstantInt::get(Op0->getType(), DefiniteZeros); auto *C = ConstantInt::get(Op0->getType(), DefiniteZeros);
return IC.replaceInstUsesWith(II, C); return IC.replaceInstUsesWith(II, C);
} }
// If the input to cttz/ctlz is known to be non-zero, // If the input to cttz/ctlz is known to be non-zero,
// then change the 'ZeroIsUndef' parameter to 'true' // then change the 'ZeroIsUndef' parameter to 'true'
// because we know the zero behavior can't affect the result. // because we know the zero behavior can't affect the result.
if (!Known.One.isNullValue() || if (!Known.One.isZero() ||
isKnownNonZero(Op0, IC.getDataLayout(), 0, &IC.getAssumptionCache(), &II, isKnownNonZero(Op0, IC.getDataLayout(), 0, &IC.getAssumptionCache(), &II,
&IC.getDominatorTree())) { &IC.getDominatorTree())) {
if (!match(II.getArgOperand(1), m_One())) if (!match(II.getArgOperand(1), m_One()))
return IC.replaceOperand(II, 1, IC.Builder.getTrue()); return IC.replaceOperand(II, 1, IC.Builder.getTrue());
} }
// Add range metadata since known bits can't completely reflect what we know. // Add range metadata since known bits can't completely reflect what we know.
// TODO: Handle splat vectors. // TODO: Handle splat vectors.
▲ Show 20 Lines • Show All 2,746 Lines • Show Last 20 Lines

llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp

Show First 20 Lines • Show All 982 Lines • ▼ Show 20 LinesInstruction *InstCombinerImpl::transformZExtICmp(ICmpInst *Cmp, ZExtInst &Zext) {
// If we are just checking for a icmp eq of a single bit and zext'ing it // If we are just checking for a icmp eq of a single bit and zext'ing it
// to an integer, then shift the bit to the appropriate place and then // to an integer, then shift the bit to the appropriate place and then
// cast to integer to avoid the comparison. // cast to integer to avoid the comparison.
const APInt *Op1CV; const APInt *Op1CV;
if (match(Cmp->getOperand(1), m_APInt(Op1CV))) { if (match(Cmp->getOperand(1), m_APInt(Op1CV))) {
// zext (x <s 0) to i32 --> x>>u31 true if signbit set. // zext (x <s 0) to i32 --> x>>u31 true if signbit set.
// zext (x >s -1) to i32 --> (x>>u31)^1 true if signbit clear. // zext (x >s -1) to i32 --> (x>>u31)^1 true if signbit clear.
if ((Cmp->getPredicate() == ICmpInst::ICMP_SLT && Op1CV->isNullValue()) || if ((Cmp->getPredicate() == ICmpInst::ICMP_SLT && Op1CV->isZero()) ||
(Cmp->getPredicate() == ICmpInst::ICMP_SGT && Op1CV->isAllOnesValue())) { (Cmp->getPredicate() == ICmpInst::ICMP_SGT && Op1CV->isAllOnes())) {
Value *In = Cmp->getOperand(0); Value *In = Cmp->getOperand(0);
Value *Sh = ConstantInt::get(In->getType(), Value *Sh = ConstantInt::get(In->getType(),
In->getType()->getScalarSizeInBits() - 1); In->getType()->getScalarSizeInBits() - 1);
In = Builder.CreateLShr(In, Sh, In->getName() + ".lobit"); In = Builder.CreateLShr(In, Sh, In->getName() + ".lobit");
if (In->getType() != Zext.getType()) if (In->getType() != Zext.getType())
In = Builder.CreateIntCast(In, Zext.getType(), false /*ZExt*/); In = Builder.CreateIntCast(In, Zext.getType(), false /*ZExt*/);
if (Cmp->getPredicate() == ICmpInst::ICMP_SGT) { if (Cmp->getPredicate() == ICmpInst::ICMP_SGT) {
Constant *One = ConstantInt::get(In->getType(), 1); Constant *One = ConstantInt::get(In->getType(), 1);
In = Builder.CreateXor(In, One, In->getName() + ".not"); In = Builder.CreateXor(In, One, In->getName() + ".not");
} }
return replaceInstUsesWith(Zext, In); return replaceInstUsesWith(Zext, In);
} }
// zext (X == 0) to i32 --> X^1 iff X has only the low bit set. // zext (X == 0) to i32 --> X^1 iff X has only the low bit set.
// zext (X == 0) to i32 --> (X>>1)^1 iff X has only the 2nd bit set. // zext (X == 0) to i32 --> (X>>1)^1 iff X has only the 2nd bit set.
// zext (X == 1) to i32 --> X iff X has only the low bit set. // zext (X == 1) to i32 --> X iff X has only the low bit set.
// zext (X == 2) to i32 --> X>>1 iff X has only the 2nd bit set. // zext (X == 2) to i32 --> X>>1 iff X has only the 2nd bit set.
// zext (X != 0) to i32 --> X iff X has only the low bit set. // zext (X != 0) to i32 --> X iff X has only the low bit set.
// zext (X != 0) to i32 --> X>>1 iff X has only the 2nd bit set. // zext (X != 0) to i32 --> X>>1 iff X has only the 2nd bit set.
// zext (X != 1) to i32 --> X^1 iff X has only the low bit set. // zext (X != 1) to i32 --> X^1 iff X has only the low bit set.
// zext (X != 2) to i32 --> (X>>1)^1 iff X has only the 2nd bit set. // zext (X != 2) to i32 --> (X>>1)^1 iff X has only the 2nd bit set.
if ((Op1CV->isNullValue() || Op1CV->isPowerOf2()) && if ((Op1CV->isZero() || Op1CV->isPowerOf2()) &&
// This only works for EQ and NE // This only works for EQ and NE
Cmp->isEquality()) { Cmp->isEquality()) {
// If Op1C some other power of two, convert: // If Op1C some other power of two, convert:
KnownBits Known = computeKnownBits(Cmp->getOperand(0), 0, &Zext); KnownBits Known = computeKnownBits(Cmp->getOperand(0), 0, &Zext);
APInt KnownZeroMask(~Known.Zero); APInt KnownZeroMask(~Known.Zero);
if (KnownZeroMask.isPowerOf2()) { // Exactly 1 possible 1? if (KnownZeroMask.isPowerOf2()) { // Exactly 1 possible 1?
bool isNE = Cmp->getPredicate() == ICmpInst::ICMP_NE; bool isNE = Cmp->getPredicate() == ICmpInst::ICMP_NE;
if (!Op1CV->isNullValue() && (*Op1CV != KnownZeroMask)) { if (!Op1CV->isZero() && (*Op1CV != KnownZeroMask)) {
// (X&4) == 2 --> false // (X&4) == 2 --> false
// (X&4) != 2 --> true // (X&4) != 2 --> true
Constant *Res = ConstantInt::get(Zext.getType(), isNE); Constant *Res = ConstantInt::get(Zext.getType(), isNE);
return replaceInstUsesWith(Zext, Res); return replaceInstUsesWith(Zext, Res);
} }
uint32_t ShAmt = KnownZeroMask.logBase2(); uint32_t ShAmt = KnownZeroMask.logBase2();
Value *In = Cmp->getOperand(0); Value *In = Cmp->getOperand(0);
if (ShAmt) { if (ShAmt) {
// Perform a logical shr by shiftamt. // Perform a logical shr by shiftamt.
// Insert the shift to put the result in the low bit. // Insert the shift to put the result in the low bit.
In = Builder.CreateLShr(In, ConstantInt::get(In->getType(), ShAmt), In = Builder.CreateLShr(In, ConstantInt::get(In->getType(), ShAmt),
In->getName() + ".lobit"); In->getName() + ".lobit");
} }
if (!Op1CV->isNullValue() == isNE) { // Toggle the low bit. if (!Op1CV->isZero() == isNE) { // Toggle the low bit.
Constant *One = ConstantInt::get(In->getType(), 1); Constant *One = ConstantInt::get(In->getType(), 1);
In = Builder.CreateXor(In, One); In = Builder.CreateXor(In, One);
} }
if (Zext.getType() == In->getType()) if (Zext.getType() == In->getType())
return replaceInstUsesWith(Zext, In); return replaceInstUsesWith(Zext, In);
Value *IntCast = Builder.CreateIntCast(In, Zext.getType(), false); Value *IntCast = Builder.CreateIntCast(In, Zext.getType(), false);
▲ Show 20 Lines • Show All 1,802 Lines • Show Last 20 Lines

llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 72 Lines • ▼ Show 20 Lines
/// Returns true if the exploded icmp can be expressed as a signed comparison /// Returns true if the exploded icmp can be expressed as a signed comparison
/// to zero and updates the predicate accordingly. /// to zero and updates the predicate accordingly.
/// The signedness of the comparison is preserved. /// The signedness of the comparison is preserved.
/// TODO: Refactor with decomposeBitTestICmp()? /// TODO: Refactor with decomposeBitTestICmp()?
static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) { static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) {
if (!ICmpInst::isSigned(Pred)) if (!ICmpInst::isSigned(Pred))
return false; return false;
if (C.isNullValue()) if (C.isZero())
return ICmpInst::isRelational(Pred); return ICmpInst::isRelational(Pred);
if (C.isOneValue()) { if (C.isOne()) {
if (Pred == ICmpInst::ICMP_SLT) { if (Pred == ICmpInst::ICMP_SLT) {
Pred = ICmpInst::ICMP_SLE; Pred = ICmpInst::ICMP_SLE;
return true; return true;
} }
} else if (C.isAllOnesValue()) { } else if (C.isAllOnes()) {
if (Pred == ICmpInst::ICMP_SGT) { if (Pred == ICmpInst::ICMP_SGT) {
Pred = ICmpInst::ICMP_SGE; Pred = ICmpInst::ICMP_SGE;
return true; return true;
} }
} }
return false; return false;
} }
▲ Show 20 Lines • Show All 1,044 Lines • ▼ Show 20 LinesInstruction *InstCombinerImpl::foldICmpShrConstConst(ICmpInst &I, Value *A,
auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
if (I.getPredicate() == I.ICMP_NE) if (I.getPredicate() == I.ICMP_NE)
Pred = CmpInst::getInversePredicate(Pred); Pred = CmpInst::getInversePredicate(Pred);
return new ICmpInst(Pred, LHS, RHS); return new ICmpInst(Pred, LHS, RHS);
}; };
// Don't bother doing any work for cases which InstSimplify handles. // Don't bother doing any work for cases which InstSimplify handles.
if (AP2.isNullValue()) if (AP2.isZero())
return nullptr; return nullptr;
bool IsAShr = isa<AShrOperator>(I.getOperand(0)); bool IsAShr = isa<AShrOperator>(I.getOperand(0));
if (IsAShr) { if (IsAShr) {
if (AP2.isAllOnesValue()) if (AP2.isAllOnes())
return nullptr; return nullptr;
if (AP2.isNegative() != AP1.isNegative()) if (AP2.isNegative() != AP1.isNegative())
return nullptr; return nullptr;
if (AP2.sgt(AP1)) if (AP2.sgt(AP1))
return nullptr; return nullptr;
} }
if (!AP1) if (!AP1)
Show All 9 Lines if (IsAShr && AP1.isNegative())
Shift = AP1.countLeadingOnes() - AP2.countLeadingOnes(); Shift = AP1.countLeadingOnes() - AP2.countLeadingOnes();
else else
Shift = AP1.countLeadingZeros() - AP2.countLeadingZeros(); Shift = AP1.countLeadingZeros() - AP2.countLeadingZeros();
if (Shift > 0) { if (Shift > 0) {
if (IsAShr && AP1 == AP2.ashr(Shift)) { if (IsAShr && AP1 == AP2.ashr(Shift)) {
// There are multiple solutions if we are comparing against -1 and the LHS // There are multiple solutions if we are comparing against -1 and the LHS
// of the ashr is not a power of two. // of the ashr is not a power of two.
if (AP1.isAllOnesValue() && !AP2.isPowerOf2()) if (AP1.isAllOnes() && !AP2.isPowerOf2())
return getICmp(I.ICMP_UGE, A, ConstantInt::get(A->getType(), Shift)); return getICmp(I.ICMP_UGE, A, ConstantInt::get(A->getType(), Shift));
return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
} else if (AP1 == AP2.lshr(Shift)) { } else if (AP1 == AP2.lshr(Shift)) {
return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
} }
} }
// Shifting const2 will never be equal to const1. // Shifting const2 will never be equal to const1.
Show All 11 LinesInstruction *InstCombinerImpl::foldICmpShlConstConst(ICmpInst &I, Value *A,
auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
if (I.getPredicate() == I.ICMP_NE) if (I.getPredicate() == I.ICMP_NE)
Pred = CmpInst::getInversePredicate(Pred); Pred = CmpInst::getInversePredicate(Pred);
return new ICmpInst(Pred, LHS, RHS); return new ICmpInst(Pred, LHS, RHS);
}; };
// Don't bother doing any work for cases which InstSimplify handles. // Don't bother doing any work for cases which InstSimplify handles.
if (AP2.isNullValue()) if (AP2.isZero())
return nullptr; return nullptr;
unsigned AP2TrailingZeros = AP2.countTrailingZeros(); unsigned AP2TrailingZeros = AP2.countTrailingZeros();
if (!AP1 && AP2TrailingZeros != 0) if (!AP1 && AP2TrailingZeros != 0)
return getICmp( return getICmp(
I.ICMP_UGE, A, I.ICMP_UGE, A,
ConstantInt::get(A->getType(), AP2.getBitWidth() - AP2TrailingZeros)); ConstantInt::get(A->getType(), AP2.getBitWidth() - AP2TrailingZeros));
▲ Show 20 Lines • Show All 321 Lines • ▼ Show 20 Lines
} }
/// Fold icmp (trunc X, Y), C. /// Fold icmp (trunc X, Y), C.
Instruction *InstCombinerImpl::foldICmpTruncConstant(ICmpInst &Cmp, Instruction *InstCombinerImpl::foldICmpTruncConstant(ICmpInst &Cmp,
TruncInst *Trunc, TruncInst *Trunc,
const APInt &C) { const APInt &C) {
ICmpInst::Predicate Pred = Cmp.getPredicate(); ICmpInst::Predicate Pred = Cmp.getPredicate();
Value *X = Trunc->getOperand(0); Value *X = Trunc->getOperand(0);
if (C.isOneValue() && C.getBitWidth() > 1) { if (C.isOne() && C.getBitWidth() > 1) {
// icmp slt trunc(signum(V)) 1 --> icmp slt V, 1 // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1
Value *V = nullptr; Value *V = nullptr;
if (Pred == ICmpInst::ICMP_SLT && match(X, m_Signum(m_Value(V)))) if (Pred == ICmpInst::ICMP_SLT && match(X, m_Signum(m_Value(V))))
return new ICmpInst(ICmpInst::ICMP_SLT, V, return new ICmpInst(ICmpInst::ICMP_SLT, V,
ConstantInt::get(V->getType(), 1)); ConstantInt::get(V->getType(), 1));
} }
unsigned DstBits = Trunc->getType()->getScalarSizeInBits(), unsigned DstBits = Trunc->getType()->getScalarSizeInBits(),
▲ Show 20 Lines • Show All 164 Lines • ▼ Show 20 Lines if (AnyCmpCstBitsShiftedOut) {
return new ICmpInst(Cmp.getPredicate(), return new ICmpInst(Cmp.getPredicate(),
NewAnd, ConstantInt::get(And->getType(), NewCmpCst)); NewAnd, ConstantInt::get(And->getType(), NewCmpCst));
} }
} }
// Turn ((X >> Y) & C2) == 0 into (X & (C2 << Y)) == 0. The latter is // Turn ((X >> Y) & C2) == 0 into (X & (C2 << Y)) == 0. The latter is
// preferable because it allows the C2 << Y expression to be hoisted out of a // preferable because it allows the C2 << Y expression to be hoisted out of a
// loop if Y is invariant and X is not. // loop if Y is invariant and X is not.
if (Shift->hasOneUse() && C1.isNullValue() && Cmp.isEquality() && if (Shift->hasOneUse() && C1.isZero() && Cmp.isEquality() &&
!Shift->isArithmeticShift() && !isa<Constant>(Shift->getOperand(0))) { !Shift->isArithmeticShift() && !isa<Constant>(Shift->getOperand(0))) {
// Compute C2 << Y. // Compute C2 << Y.
Value *NewShift = Value *NewShift =
IsShl ? Builder.CreateLShr(And->getOperand(1), Shift->getOperand(1)) IsShl ? Builder.CreateLShr(And->getOperand(1), Shift->getOperand(1))
: Builder.CreateShl(And->getOperand(1), Shift->getOperand(1)); : Builder.CreateShl(And->getOperand(1), Shift->getOperand(1));
// Compute X & (C2 << Y). // Compute X & (C2 << Y).
Value *NewAnd = Builder.CreateAnd(Shift->getOperand(0), NewShift); Value *NewAnd = Builder.CreateAnd(Shift->getOperand(0), NewShift);
return replaceOperand(Cmp, 0, NewAnd); return replaceOperand(Cmp, 0, NewAnd);
} }
return nullptr; return nullptr;
} }
/// Fold icmp (and X, C2), C1. /// Fold icmp (and X, C2), C1.
Instruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp, Instruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp,
BinaryOperator *And, BinaryOperator *And,
const APInt &C1) { const APInt &C1) {
bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE; bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE;
// For vectors: icmp ne (and X, 1), 0 --> trunc X to N x i1 // For vectors: icmp ne (and X, 1), 0 --> trunc X to N x i1
// TODO: We canonicalize to the longer form for scalars because we have // TODO: We canonicalize to the longer form for scalars because we have
// better analysis/folds for icmp, and codegen may be better with icmp. // better analysis/folds for icmp, and codegen may be better with icmp.
if (isICMP_NE && Cmp.getType()->isVectorTy() && C1.isNullValue() && if (isICMP_NE && Cmp.getType()->isVectorTy() && C1.isZero() &&
match(And->getOperand(1), m_One())) match(And->getOperand(1), m_One()))
return new TruncInst(And->getOperand(0), Cmp.getType()); return new TruncInst(And->getOperand(0), Cmp.getType());
const APInt *C2; const APInt *C2;
Value *X; Value *X;
if (!match(And, m_And(m_Value(X), m_APInt(C2)))) if (!match(And, m_And(m_Value(X), m_APInt(C2))))
return nullptr; return nullptr;
// Don't perform the following transforms if the AND has multiple uses // Don't perform the following transforms if the AND has multiple uses
if (!And->hasOneUse()) if (!And->hasOneUse())
return nullptr; return nullptr;
if (Cmp.isEquality() && C1.isNullValue()) { if (Cmp.isEquality() && C1.isZero()) {
// Restrict this fold to single-use 'and' (PR10267). // Restrict this fold to single-use 'and' (PR10267).
// Replace (and X, (1 << size(X)-1) != 0) with X s< 0 // Replace (and X, (1 << size(X)-1) != 0) with X s< 0
if (C2->isSignMask()) { if (C2->isSignMask()) {
Constant *Zero = Constant::getNullValue(X->getType()); Constant *Zero = Constant::getNullValue(X->getType());
auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE; auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE;
return new ICmpInst(NewPred, X, Zero); return new ICmpInst(NewPred, X, Zero);
} }
Show All 33 LinesInstruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp,
if (Instruction *I = foldICmpAndShift(Cmp, And, C1, *C2)) if (Instruction *I = foldICmpAndShift(Cmp, And, C1, *C2))
return I; return I;
// (icmp pred (and (or (lshr A, B), A), 1), 0) --> // (icmp pred (and (or (lshr A, B), A), 1), 0) -->
// (icmp pred (and A, (or (shl 1, B), 1), 0)) // (icmp pred (and A, (or (shl 1, B), 1), 0))
// //
// iff pred isn't signed // iff pred isn't signed
if (!Cmp.isSigned() && C1.isNullValue() && And->getOperand(0)->hasOneUse() && if (!Cmp.isSigned() && C1.isZero() && And->getOperand(0)->hasOneUse() &&
match(And->getOperand(1), m_One())) { match(And->getOperand(1), m_One())) {
Constant *One = cast<Constant>(And->getOperand(1)); Constant *One = cast<Constant>(And->getOperand(1));
Value *Or = And->getOperand(0); Value *Or = And->getOperand(0);
Value *A, *B, *LShr; Value *A, *B, *LShr;
if (match(Or, m_Or(m_Value(LShr), m_Value(A))) && if (match(Or, m_Or(m_Value(LShr), m_Value(A))) &&
match(LShr, m_LShr(m_Specific(A), m_Value(B)))) { match(LShr, m_LShr(m_Specific(A), m_Value(B)))) {
unsigned UsesRemoved = 0; unsigned UsesRemoved = 0;
if (And->hasOneUse()) if (And->hasOneUse())
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines auto NewPred =
Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGT : CmpInst::ICMP_ULE; Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGT : CmpInst::ICMP_ULE;
return new ICmpInst(NewPred, X, SubOne(cast<Constant>(Cmp.getOperand(1)))); return new ICmpInst(NewPred, X, SubOne(cast<Constant>(Cmp.getOperand(1))));
} }
// (X & C2) == 0 -> (trunc X) >= 0 // (X & C2) == 0 -> (trunc X) >= 0
// (X & C2) != 0 -> (trunc X) < 0 // (X & C2) != 0 -> (trunc X) < 0
// iff C2 is a power of 2 and it masks the sign bit of a legal integer type. // iff C2 is a power of 2 and it masks the sign bit of a legal integer type.
const APInt *C2; const APInt *C2;
if (And->hasOneUse() && C.isNullValue() && match(Y, m_APInt(C2))) { if (And->hasOneUse() && C.isZero() && match(Y, m_APInt(C2))) {
int32_t ExactLogBase2 = C2->exactLogBase2(); int32_t ExactLogBase2 = C2->exactLogBase2();
if (ExactLogBase2 != -1 && DL.isLegalInteger(ExactLogBase2 + 1)) { if (ExactLogBase2 != -1 && DL.isLegalInteger(ExactLogBase2 + 1)) {
Type *NTy = IntegerType::get(Cmp.getContext(), ExactLogBase2 + 1); Type *NTy = IntegerType::get(Cmp.getContext(), ExactLogBase2 + 1);
if (auto *AndVTy = dyn_cast<VectorType>(And->getType())) if (auto *AndVTy = dyn_cast<VectorType>(And->getType()))
NTy = VectorType::get(NTy, AndVTy->getElementCount()); NTy = VectorType::get(NTy, AndVTy->getElementCount());
Value *Trunc = Builder.CreateTrunc(X, NTy); Value *Trunc = Builder.CreateTrunc(X, NTy);
auto NewPred = auto NewPred =
Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_SGE : CmpInst::ICMP_SLT; Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_SGE : CmpInst::ICMP_SLT;
return new ICmpInst(NewPred, Trunc, Constant::getNullValue(NTy)); return new ICmpInst(NewPred, Trunc, Constant::getNullValue(NTy));
} }
} }
return nullptr; return nullptr;
} }
/// Fold icmp (or X, Y), C. /// Fold icmp (or X, Y), C.
Instruction *InstCombinerImpl::foldICmpOrConstant(ICmpInst &Cmp, Instruction *InstCombinerImpl::foldICmpOrConstant(ICmpInst &Cmp,
BinaryOperator *Or, BinaryOperator *Or,
const APInt &C) { const APInt &C) {
ICmpInst::Predicate Pred = Cmp.getPredicate(); ICmpInst::Predicate Pred = Cmp.getPredicate();
if (C.isOneValue()) { if (C.isOne()) {
// icmp slt signum(V) 1 --> icmp slt V, 1 // icmp slt signum(V) 1 --> icmp slt V, 1
Value *V = nullptr; Value *V = nullptr;
if (Pred == ICmpInst::ICMP_SLT && match(Or, m_Signum(m_Value(V)))) if (Pred == ICmpInst::ICMP_SLT && match(Or, m_Signum(m_Value(V))))
return new ICmpInst(ICmpInst::ICMP_SLT, V, return new ICmpInst(ICmpInst::ICMP_SLT, V,
ConstantInt::get(V->getType(), 1)); ConstantInt::get(V->getType(), 1));
} }
Value *OrOp0 = Or->getOperand(0), *OrOp1 = Or->getOperand(1); Value *OrOp0 = Or->getOperand(0), *OrOp1 = Or->getOperand(1);
Show All 13 Lines if (match(OrOp1, m_APInt(MaskC)) && Cmp.isEquality()) {
// (X | MaskC) != C --> (X & ~MaskC) != C ^ MaskC // (X | MaskC) != C --> (X & ~MaskC) != C ^ MaskC
if (Or->hasOneUse()) { if (Or->hasOneUse()) {
Value *And = Builder.CreateAnd(OrOp0, ~(*MaskC)); Value *And = Builder.CreateAnd(OrOp0, ~(*MaskC));
Constant *NewC = ConstantInt::get(Or->getType(), C ^ (*MaskC)); Constant *NewC = ConstantInt::get(Or->getType(), C ^ (*MaskC));
return new ICmpInst(Pred, And, NewC); return new ICmpInst(Pred, And, NewC);
} }
} }
if (!Cmp.isEquality() || !C.isNullValue() || !Or->hasOneUse()) if (!Cmp.isEquality() || !C.isZero() || !Or->hasOneUse())
return nullptr; return nullptr;
Value *P, *Q; Value *P, *Q;
if (match(Or, m_Or(m_PtrToInt(m_Value(P)), m_PtrToInt(m_Value(Q))))) { if (match(Or, m_Or(m_PtrToInt(m_Value(P)), m_PtrToInt(m_Value(Q))))) {
// Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0 // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0
// -> and (icmp eq P, null), (icmp eq Q, null). // -> and (icmp eq P, null), (icmp eq Q, null).
Value *CmpP = Value *CmpP =
Builder.CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType())); Builder.CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType()));
Show All 34 Lines if (isSignTest(Pred, C) && Mul->hasNoSignedWrap()) {
if (MulC->isNegative()) if (MulC->isNegative())
Pred = ICmpInst::getSwappedPredicate(Pred); Pred = ICmpInst::getSwappedPredicate(Pred);
return new ICmpInst(Pred, Mul->getOperand(0), return new ICmpInst(Pred, Mul->getOperand(0),
Constant::getNullValue(Mul->getType())); Constant::getNullValue(Mul->getType()));
} }
// If the multiply does not wrap, try to divide the compare constant by the // If the multiply does not wrap, try to divide the compare constant by the
// multiplication factor. // multiplication factor.
if (Cmp.isEquality() && !MulC->isNullValue()) { if (Cmp.isEquality() && !MulC->isZero()) {
// (mul nsw X, MulC) == C --> X == C /s MulC // (mul nsw X, MulC) == C --> X == C /s MulC
if (Mul->hasNoSignedWrap() && C.srem(*MulC).isNullValue()) { if (Mul->hasNoSignedWrap() && C.srem(*MulC).isZero()) {
Constant *NewC = ConstantInt::get(Mul->getType(), C.sdiv(*MulC)); Constant *NewC = ConstantInt::get(Mul->getType(), C.sdiv(*MulC));
return new ICmpInst(Pred, Mul->getOperand(0), NewC); return new ICmpInst(Pred, Mul->getOperand(0), NewC);
} }
// (mul nuw X, MulC) == C --> X == C /u MulC // (mul nuw X, MulC) == C --> X == C /u MulC
if (Mul->hasNoUnsignedWrap() && C.urem(*MulC).isNullValue()) { if (Mul->hasNoUnsignedWrap() && C.urem(*MulC).isZero()) {
Constant *NewC = ConstantInt::get(Mul->getType(), C.udiv(*MulC)); Constant *NewC = ConstantInt::get(Mul->getType(), C.udiv(*MulC));
return new ICmpInst(Pred, Mul->getOperand(0), NewC); return new ICmpInst(Pred, Mul->getOperand(0), NewC);
} }
} }
return nullptr; return nullptr;
} }
Show All 28 Lines if (CLog2 == TypeBits - 1) {
if (Pred == ICmpInst::ICMP_UGE) if (Pred == ICmpInst::ICMP_UGE)
Pred = ICmpInst::ICMP_EQ; Pred = ICmpInst::ICMP_EQ;
else if (Pred == ICmpInst::ICMP_ULT) else if (Pred == ICmpInst::ICMP_ULT)
Pred = ICmpInst::ICMP_NE; Pred = ICmpInst::ICMP_NE;
} }
return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, CLog2)); return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, CLog2));
} else if (Cmp.isSigned()) { } else if (Cmp.isSigned()) {
Constant *BitWidthMinusOne = ConstantInt::get(ShiftType, TypeBits - 1); Constant *BitWidthMinusOne = ConstantInt::get(ShiftType, TypeBits - 1);
if (C.isAllOnesValue()) { if (C.isAllOnes()) {
// (1 << Y) <= -1 -> Y == 31 // (1 << Y) <= -1 -> Y == 31
if (Pred == ICmpInst::ICMP_SLE) if (Pred == ICmpInst::ICMP_SLE)
return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne); return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne);
// (1 << Y) > -1 -> Y != 31 // (1 << Y) > -1 -> Y != 31
if (Pred == ICmpInst::ICMP_SGT) if (Pred == ICmpInst::ICMP_SGT)
return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne); return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne);
} else if (!C) { } else if (!C) {
▲ Show 20 Lines • Show All 157 Lines • ▼ Show 20 Lines
/// Fold icmp ({al}shr X, Y), C. /// Fold icmp ({al}shr X, Y), C.
Instruction *InstCombinerImpl::foldICmpShrConstant(ICmpInst &Cmp, Instruction *InstCombinerImpl::foldICmpShrConstant(ICmpInst &Cmp,
BinaryOperator *Shr, BinaryOperator *Shr,
const APInt &C) { const APInt &C) {
// An exact shr only shifts out zero bits, so: // An exact shr only shifts out zero bits, so:
// icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0 // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0
Value *X = Shr->getOperand(0); Value *X = Shr->getOperand(0);
CmpInst::Predicate Pred = Cmp.getPredicate(); CmpInst::Predicate Pred = Cmp.getPredicate();
if (Cmp.isEquality() && Shr->isExact() && Shr->hasOneUse() && if (Cmp.isEquality() && Shr->isExact() && Shr->hasOneUse() && C.isZero())
C.isNullValue())
return new ICmpInst(Pred, X, Cmp.getOperand(1)); return new ICmpInst(Pred, X, Cmp.getOperand(1));
const APInt *ShiftVal; const APInt *ShiftVal;
if (Cmp.isEquality() && match(Shr->getOperand(0), m_APInt(ShiftVal))) if (Cmp.isEquality() && match(Shr->getOperand(0), m_APInt(ShiftVal)))
return foldICmpShrConstConst(Cmp, Shr->getOperand(1), C, *ShiftVal); return foldICmpShrConstConst(Cmp, Shr->getOperand(1), C, *ShiftVal);
const APInt *ShiftAmt; const APInt *ShiftAmt;
if (!match(Shr->getOperand(1), m_APInt(ShiftAmt))) if (!match(Shr->getOperand(1), m_APInt(ShiftAmt)))
▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Lines assert(((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) ||
(!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) &&
"Expected icmp+shr simplify did not occur."); "Expected icmp+shr simplify did not occur.");
// If the bits shifted out are known zero, compare the unshifted value: // If the bits shifted out are known zero, compare the unshifted value:
// (X & 4) >> 1 == 2 --> (X & 4) == 4. // (X & 4) >> 1 == 2 --> (X & 4) == 4.
if (Shr->isExact()) if (Shr->isExact())
return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, C << ShAmtVal)); return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, C << ShAmtVal));
if (C.isNullValue()) { if (C.isZero()) {
// == 0 is u< 1. // == 0 is u< 1.
if (Pred == CmpInst::ICMP_EQ) if (Pred == CmpInst::ICMP_EQ)
return new ICmpInst(CmpInst::ICMP_ULT, X, return new ICmpInst(CmpInst::ICMP_ULT, X,
ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal))); ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal)));
else else
return new ICmpInst(CmpInst::ICMP_UGT, X, return new ICmpInst(CmpInst::ICMP_UGT, X,
ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal) - 1)); ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal) - 1));
} }
Show All 22 LinesInstruction *InstCombinerImpl::foldICmpSRemConstant(ICmpInst &Cmp,
// TODO: The one-use check is standard because we do not typically want to // TODO: The one-use check is standard because we do not typically want to
// create longer instruction sequences, but this might be a special-case // create longer instruction sequences, but this might be a special-case
// because srem is not good for analysis or codegen. // because srem is not good for analysis or codegen.
if (!SRem->hasOneUse()) if (!SRem->hasOneUse())
return nullptr; return nullptr;
const APInt *DivisorC; const APInt *DivisorC;
if (!C.isNullValue() || !match(SRem->getOperand(1), m_Power2(DivisorC))) if (!C.isZero() || !match(SRem->getOperand(1), m_Power2(DivisorC)))
return nullptr; return nullptr;
// Mask off the sign bit and the modulo bits (low-bits). // Mask off the sign bit and the modulo bits (low-bits).
Type *Ty = SRem->getType(); Type *Ty = SRem->getType();
APInt SignMask = APInt::getSignMask(Ty->getScalarSizeInBits()); APInt SignMask = APInt::getSignMask(Ty->getScalarSizeInBits());
Constant *MaskC = ConstantInt::get(Ty, SignMask | (*DivisorC - 1)); Constant *MaskC = ConstantInt::get(Ty, SignMask | (*DivisorC - 1));
Value *And = Builder.CreateAnd(SRem->getOperand(0), MaskC); Value *And = Builder.CreateAnd(SRem->getOperand(0), MaskC);
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 LinesInstruction *InstCombinerImpl::foldICmpDivConstant(ICmpInst &Cmp,
bool DivIsSigned = Div->getOpcode() == Instruction::SDiv; bool DivIsSigned = Div->getOpcode() == Instruction::SDiv;
if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned()) if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned())
return nullptr; return nullptr;
// The ProdOV computation fails on divide by 0 and divide by -1. Cases with // The ProdOV computation fails on divide by 0 and divide by -1. Cases with
// INT_MIN will also fail if the divisor is 1. Although folds of all these // INT_MIN will also fail if the divisor is 1. Although folds of all these
// division-by-constant cases should be present, we can not assert that they // division-by-constant cases should be present, we can not assert that they
// have happened before we reach this icmp instruction. // have happened before we reach this icmp instruction.
if (C2->isNullValue() || C2->isOneValue() || if (C2->isZero() || C2->isOne() || (DivIsSigned && C2->isAllOnes()))
(DivIsSigned && C2->isAllOnesValue()))
return nullptr; return nullptr;
// Compute Prod = C * C2. We are essentially solving an equation of // Compute Prod = C * C2. We are essentially solving an equation of
// form X / C2 = C. We solve for X by multiplying C2 and C. // form X / C2 = C. We solve for X by multiplying C2 and C.
// By solving for X, we can turn this into a range check instead of computing // By solving for X, we can turn this into a range check instead of computing
// a divide. // a divide.
APInt Prod = C * *C2; APInt Prod = C * *C2;
Show All 23 Lines if (!DivIsSigned) { // udiv
LoBound = Prod; LoBound = Prod;
HiOverflow = LoOverflow = ProdOV; HiOverflow = LoOverflow = ProdOV;
if (!HiOverflow) { if (!HiOverflow) {
// If this is not an exact divide, then many values in the range collapse // If this is not an exact divide, then many values in the range collapse
// to the same result value. // to the same result value.
HiOverflow = addWithOverflow(HiBound, LoBound, RangeSize, false); HiOverflow = addWithOverflow(HiBound, LoBound, RangeSize, false);
} }
} else if (C2->isStrictlyPositive()) { // Divisor is > 0. } else if (C2->isStrictlyPositive()) { // Divisor is > 0.
if (C.isNullValue()) { // (X / pos) op 0 if (C.isZero()) { // (X / pos) op 0
// Can't overflow. e.g. X/2 op 0 --> [-1, 2) // Can't overflow. e.g. X/2 op 0 --> [-1, 2)
LoBound = -(RangeSize - 1); LoBound = -(RangeSize - 1);
HiBound = RangeSize; HiBound = RangeSize;
} else if (C.isStrictlyPositive()) { // (X / pos) op pos } else if (C.isStrictlyPositive()) { // (X / pos) op pos
LoBound = Prod; // e.g. X/5 op 3 --> [15, 20) LoBound = Prod; // e.g. X/5 op 3 --> [15, 20)
HiOverflow = LoOverflow = ProdOV; HiOverflow = LoOverflow = ProdOV;
if (!HiOverflow) if (!HiOverflow)
HiOverflow = addWithOverflow(HiBound, Prod, RangeSize, true); HiOverflow = addWithOverflow(HiBound, Prod, RangeSize, true);
} else { // (X / pos) op neg } else { // (X / pos) op neg
// e.g. X/5 op -3 --> [-15-4, -15+1) --> [-19, -14) // e.g. X/5 op -3 --> [-15-4, -15+1) --> [-19, -14)
HiBound = Prod + 1; HiBound = Prod + 1;
LoOverflow = HiOverflow = ProdOV ? -1 : 0; LoOverflow = HiOverflow = ProdOV ? -1 : 0;
if (!LoOverflow) { if (!LoOverflow) {
APInt DivNeg = -RangeSize; APInt DivNeg = -RangeSize;
LoOverflow = addWithOverflow(LoBound, HiBound, DivNeg, true) ? -1 : 0; LoOverflow = addWithOverflow(LoBound, HiBound, DivNeg, true) ? -1 : 0;
} }
} }
} else if (C2->isNegative()) { // Divisor is < 0. } else if (C2->isNegative()) { // Divisor is < 0.
if (Div->isExact()) if (Div->isExact())
RangeSize.negate(); RangeSize.negate();
if (C.isNullValue()) { // (X / neg) op 0 if (C.isZero()) { // (X / neg) op 0
// e.g. X/-5 op 0 --> [-4, 5) // e.g. X/-5 op 0 --> [-4, 5)
LoBound = RangeSize + 1; LoBound = RangeSize + 1;
HiBound = -RangeSize; HiBound = -RangeSize;
if (HiBound == *C2) { // -INTMIN = INTMIN if (HiBound == *C2) { // -INTMIN = INTMIN
HiOverflow = 1; // [INTMIN+1, overflow) HiOverflow = 1; // [INTMIN+1, overflow)
HiBound = APInt(); // e.g. X/INTMIN = 0 --> X > INTMIN HiBound = APInt(); // e.g. X/INTMIN = 0 --> X > INTMIN
} }
} else if (C.isStrictlyPositive()) { // (X / neg) op pos } else if (C.isStrictlyPositive()) { // (X / neg) op pos
// e.g. X/-5 op 3 --> [-19, -14) // e.g. X/-5 op 3 --> [-19, -14)
HiBound = Prod + 1; HiBound = Prod + 1;
HiOverflow = LoOverflow = ProdOV ? -1 : 0; HiOverflow = LoOverflow = ProdOV ? -1 : 0;
if (!LoOverflow) if (!LoOverflow)
LoOverflow = addWithOverflow(LoBound, HiBound, RangeSize, true) ? -1:0; LoOverflow = addWithOverflow(LoBound, HiBound, RangeSize, true) ? -1:0;
} else { // (X / neg) op neg } else { // (X / neg) op neg
LoBound = Prod; // e.g. X/-5 op -3 --> [15, 20) LoBound = Prod; // e.g. X/-5 op -3 --> [15, 20)
LoOverflow = HiOverflow = ProdOV; LoOverflow = HiOverflow = ProdOV;
if (!HiOverflow) if (!HiOverflow)
HiOverflow = subWithOverflow(HiBound, Prod, RangeSize, true); HiOverflow = subWithOverflow(HiBound, Prod, RangeSize, true);
} }
// Dividing by a negative swaps the condition. LT <-> GT // Dividing by a negative swaps the condition. LT <-> GT
Pred = ICmpInst::getSwappedPredicate(Pred); Pred = ICmpInst::getSwappedPredicate(Pred);
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 LinesInstruction *InstCombinerImpl::foldICmpSubConstant(ICmpInst &Cmp,
// The following transforms are only worth it if the only user of the subtract // The following transforms are only worth it if the only user of the subtract
// is the icmp. // is the icmp.
if (!Sub->hasOneUse()) if (!Sub->hasOneUse())
return nullptr; return nullptr;
if (Sub->hasNoSignedWrap()) { if (Sub->hasNoSignedWrap()) {
// (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y) // (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y)
if (Pred == ICmpInst::ICMP_SGT && C.isAllOnesValue()) if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes())
return new ICmpInst(ICmpInst::ICMP_SGE, X, Y); return new ICmpInst(ICmpInst::ICMP_SGE, X, Y);
// (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y) // (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y)
if (Pred == ICmpInst::ICMP_SGT && C.isNullValue()) if (Pred == ICmpInst::ICMP_SGT && C.isZero())
return new ICmpInst(ICmpInst::ICMP_SGT, X, Y); return new ICmpInst(ICmpInst::ICMP_SGT, X, Y);
// (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y) // (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y)
if (Pred == ICmpInst::ICMP_SLT && C.isNullValue()) if (Pred == ICmpInst::ICMP_SLT && C.isZero())
return new ICmpInst(ICmpInst::ICMP_SLT, X, Y); return new ICmpInst(ICmpInst::ICMP_SLT, X, Y);
// (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y) // (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y)
if (Pred == ICmpInst::ICMP_SLT && C.isOneValue()) if (Pred == ICmpInst::ICMP_SLT && C.isOne())
return new ICmpInst(ICmpInst::ICMP_SLE, X, Y); return new ICmpInst(ICmpInst::ICMP_SLE, X, Y);
} }
if (!match(X, m_APInt(C2))) if (!match(X, m_APInt(C2)))
return nullptr; return nullptr;
// C2 - Y <u C -> (Y | (C - 1)) == C2 // C2 - Y <u C -> (Y | (C - 1)) == C2
// iff (C2 & (C - 1)) == C - 1 and C is a power of 2 // iff (C2 & (C - 1)) == C - 1 and C is a power of 2
▲ Show 20 Lines • Show All 296 Lines • ▼ Show 20 LinesInstruction *InstCombinerImpl::foldICmpBitCast(ICmpInst &Cmp) {
// If this is checking if all elements of a vector compare are set or not, // If this is checking if all elements of a vector compare are set or not,
// invert the casted vector equality compare and test if all compare // invert the casted vector equality compare and test if all compare
// elements are clear or not. Compare against zero is generally easier for // elements are clear or not. Compare against zero is generally easier for
// analysis and codegen. // analysis and codegen.
// icmp eq/ne (bitcast (not X) to iN), -1 --> icmp eq/ne (bitcast X to iN), 0 // icmp eq/ne (bitcast (not X) to iN), -1 --> icmp eq/ne (bitcast X to iN), 0
// Example: are all elements equal? --> are zero elements not equal? // Example: are all elements equal? --> are zero elements not equal?
// TODO: Try harder to reduce compare of 2 freely invertible operands? // TODO: Try harder to reduce compare of 2 freely invertible operands?
if (Cmp.isEquality() && C->isAllOnesValue() && Bitcast->hasOneUse() && if (Cmp.isEquality() && C->isAllOnes() && Bitcast->hasOneUse() &&
isFreeToInvert(BCSrcOp, BCSrcOp->hasOneUse())) { isFreeToInvert(BCSrcOp, BCSrcOp->hasOneUse())) {
Type *ScalarTy = Bitcast->getType(); Type *ScalarTy = Bitcast->getType();
Value *Cast = Builder.CreateBitCast(Builder.CreateNot(BCSrcOp), ScalarTy); Value *Cast = Builder.CreateBitCast(Builder.CreateNot(BCSrcOp), ScalarTy);
return new ICmpInst(Pred, Cast, ConstantInt::getNullValue(ScalarTy)); return new ICmpInst(Pred, Cast, ConstantInt::getNullValue(ScalarTy));
} }
// If this is checking if all elements of an extended vector are clear or not, // If this is checking if all elements of an extended vector are clear or not,
// compare in a narrow type to eliminate the extend: // compare in a narrow type to eliminate the extend:
// icmp eq/ne (bitcast (ext X) to iN), 0 --> icmp eq/ne (bitcast X to iM), 0 // icmp eq/ne (bitcast (ext X) to iN), 0 --> icmp eq/ne (bitcast X to iM), 0
Value *X; Value *X;
if (Cmp.isEquality() && C->isNullValue() && Bitcast->hasOneUse() && if (Cmp.isEquality() && C->isZero() && Bitcast->hasOneUse() &&
match(BCSrcOp, m_ZExtOrSExt(m_Value(X)))) { match(BCSrcOp, m_ZExtOrSExt(m_Value(X)))) {
if (auto *VecTy = dyn_cast<FixedVectorType>(X->getType())) { if (auto *VecTy = dyn_cast<FixedVectorType>(X->getType())) {
Type *NewType = Builder.getIntNTy(VecTy->getPrimitiveSizeInBits()); Type *NewType = Builder.getIntNTy(VecTy->getPrimitiveSizeInBits());
Value *NewCast = Builder.CreateBitCast(X, NewType); Value *NewCast = Builder.CreateBitCast(X, NewType);
return new ICmpInst(Pred, NewCast, ConstantInt::getNullValue(NewType)); return new ICmpInst(Pred, NewCast, ConstantInt::getNullValue(NewType));
} }
} }
▲ Show 20 Lines • Show All 124 Lines • ▼ Show 20 LinesInstruction *InstCombinerImpl::foldICmpBinOpEqualityWithConstant(
ICmpInst::Predicate Pred = Cmp.getPredicate(); ICmpInst::Predicate Pred = Cmp.getPredicate();
bool isICMP_NE = Pred == ICmpInst::ICMP_NE; bool isICMP_NE = Pred == ICmpInst::ICMP_NE;
Constant *RHS = cast<Constant>(Cmp.getOperand(1)); Constant *RHS = cast<Constant>(Cmp.getOperand(1));
Value *BOp0 = BO->getOperand(0), *BOp1 = BO->getOperand(1); Value *BOp0 = BO->getOperand(0), *BOp1 = BO->getOperand(1);
switch (BO->getOpcode()) { switch (BO->getOpcode()) {
case Instruction::SRem: case Instruction::SRem:
// If we have a signed (X % (2^c)) == 0, turn it into an unsigned one. // If we have a signed (X % (2^c)) == 0, turn it into an unsigned one.
if (C.isNullValue() && BO->hasOneUse()) { if (C.isZero() && BO->hasOneUse()) {
const APInt *BOC; const APInt *BOC;
if (match(BOp1, m_APInt(BOC)) && BOC->sgt(1) && BOC->isPowerOf2()) { if (match(BOp1, m_APInt(BOC)) && BOC->sgt(1) && BOC->isPowerOf2()) {
Value *NewRem = Builder.CreateURem(BOp0, BOp1, BO->getName()); Value *NewRem = Builder.CreateURem(BOp0, BOp1, BO->getName());
return new ICmpInst(Pred, NewRem, return new ICmpInst(Pred, NewRem,
Constant::getNullValue(BO->getType())); Constant::getNullValue(BO->getType()));
} }
} }
break; break;
case Instruction::Add: { case Instruction::Add: {
// Replace ((add A, B) != C) with (A != C-B) if B & C are constants. // Replace ((add A, B) != C) with (A != C-B) if B & C are constants.
if (Constant *BOC = dyn_cast<Constant>(BOp1)) { if (Constant *BOC = dyn_cast<Constant>(BOp1)) {
if (BO->hasOneUse()) if (BO->hasOneUse())
return new ICmpInst(Pred, BOp0, ConstantExpr::getSub(RHS, BOC)); return new ICmpInst(Pred, BOp0, ConstantExpr::getSub(RHS, BOC));
} else if (C.isNullValue()) { } else if (C.isZero()) {
// Replace ((add A, B) != 0) with (A != -B) if A or B is // Replace ((add A, B) != 0) with (A != -B) if A or B is
// efficiently invertible, or if the add has just this one use. // efficiently invertible, or if the add has just this one use.
if (Value *NegVal = dyn_castNegVal(BOp1)) if (Value *NegVal = dyn_castNegVal(BOp1))
return new ICmpInst(Pred, BOp0, NegVal); return new ICmpInst(Pred, BOp0, NegVal);
if (Value *NegVal = dyn_castNegVal(BOp0)) if (Value *NegVal = dyn_castNegVal(BOp0))
return new ICmpInst(Pred, NegVal, BOp1); return new ICmpInst(Pred, NegVal, BOp1);
if (BO->hasOneUse()) { if (BO->hasOneUse()) {
Value *Neg = Builder.CreateNeg(BOp1); Value *Neg = Builder.CreateNeg(BOp1);
Neg->takeName(BO); Neg->takeName(BO);
return new ICmpInst(Pred, BOp0, Neg); return new ICmpInst(Pred, BOp0, Neg);
} }
} }
break; break;
} }
case Instruction::Xor: case Instruction::Xor:
if (BO->hasOneUse()) { if (BO->hasOneUse()) {
if (Constant *BOC = dyn_cast<Constant>(BOp1)) { if (Constant *BOC = dyn_cast<Constant>(BOp1)) {
// For the xor case, we can xor two constants together, eliminating // For the xor case, we can xor two constants together, eliminating
// the explicit xor. // the explicit xor.
return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC)); return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC));
} else if (C.isNullValue()) { } else if (C.isZero()) {
// Replace ((xor A, B) != 0) with (A != B) // Replace ((xor A, B) != 0) with (A != B)
return new ICmpInst(Pred, BOp0, BOp1); return new ICmpInst(Pred, BOp0, BOp1);
} }
} }
break; break;
case Instruction::Sub: case Instruction::Sub:
if (BO->hasOneUse()) { if (BO->hasOneUse()) {
// Only check for constant LHS here, as constant RHS will be canonicalized // Only check for constant LHS here, as constant RHS will be canonicalized
// to add and use the fold above. // to add and use the fold above.
if (Constant *BOC = dyn_cast<Constant>(BOp0)) { if (Constant *BOC = dyn_cast<Constant>(BOp0)) {
// Replace ((sub BOC, B) != C) with (B != BOC-C). // Replace ((sub BOC, B) != C) with (B != BOC-C).
return new ICmpInst(Pred, BOp1, ConstantExpr::getSub(BOC, RHS)); return new ICmpInst(Pred, BOp1, ConstantExpr::getSub(BOC, RHS));
} else if (C.isNullValue()) { } else if (C.isZero()) {
// Replace ((sub A, B) != 0) with (A != B). // Replace ((sub A, B) != 0) with (A != B).
return new ICmpInst(Pred, BOp0, BOp1); return new ICmpInst(Pred, BOp0, BOp1);
} }
} }
break; break;
case Instruction::Or: { case Instruction::Or: {
const APInt *BOC; const APInt *BOC;
if (match(BOp1, m_APInt(BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) { if (match(BOp1, m_APInt(BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) {
Show All 12 Lines if (match(BOp1, m_APInt(BOC))) {
// If we have ((X & C) == C), turn it into ((X & C) != 0). // If we have ((X & C) == C), turn it into ((X & C) != 0).
if (C == *BOC && C.isPowerOf2()) if (C == *BOC && C.isPowerOf2())
return new ICmpInst(isICMP_NE ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE, return new ICmpInst(isICMP_NE ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE,
BO, Constant::getNullValue(RHS->getType())); BO, Constant::getNullValue(RHS->getType()));
} }
break; break;
} }
case Instruction::UDiv: case Instruction::UDiv:
if (C.isNullValue()) { if (C.isZero()) {
// (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A) // (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A)
auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT;
return new ICmpInst(NewPred, BOp1, BOp0); return new ICmpInst(NewPred, BOp1, BOp0);
} }
break; break;
default: default:
break; break;
} }
return nullptr; return nullptr;
} }
/// Fold an equality icmp with LLVM intrinsic and constant operand. /// Fold an equality icmp with LLVM intrinsic and constant operand.
Instruction *InstCombinerImpl::foldICmpEqIntrinsicWithConstant( Instruction *InstCombinerImpl::foldICmpEqIntrinsicWithConstant(
ICmpInst &Cmp, IntrinsicInst *II, const APInt &C) { ICmpInst &Cmp, IntrinsicInst *II, const APInt &C) {
Type *Ty = II->getType(); Type *Ty = II->getType();
unsigned BitWidth = C.getBitWidth(); unsigned BitWidth = C.getBitWidth();
const ICmpInst::Predicate Pred = Cmp.getPredicate(); const ICmpInst::Predicate Pred = Cmp.getPredicate();
switch (II->getIntrinsicID()) { switch (II->getIntrinsicID()) {
case Intrinsic::abs: case Intrinsic::abs:
// abs(A) == 0 -> A == 0 // abs(A) == 0 -> A == 0
// abs(A) == INT_MIN -> A == INT_MIN // abs(A) == INT_MIN -> A == INT_MIN
if (C.isNullValue() || C.isMinSignedValue()) if (C.isZero() || C.isMinSignedValue())
return new ICmpInst(Pred, II->getArgOperand(0), ConstantInt::get(Ty, C)); return new ICmpInst(Pred, II->getArgOperand(0), ConstantInt::get(Ty, C));
break; break;
case Intrinsic::bswap: case Intrinsic::bswap:
// bswap(A) == C -> A == bswap(C) // bswap(A) == C -> A == bswap(C)
return new ICmpInst(Pred, II->getArgOperand(0), return new ICmpInst(Pred, II->getArgOperand(0),
ConstantInt::get(Ty, C.byteSwap())); ConstantInt::get(Ty, C.byteSwap()));
Show All 19 Lines if (Num != BitWidth && II->hasOneUse()) {
ConstantInt::get(Ty, Mask2)); ConstantInt::get(Ty, Mask2));
} }
break; break;
} }
case Intrinsic::ctpop: { case Intrinsic::ctpop: {
// popcount(A) == 0 -> A == 0 and likewise for != // popcount(A) == 0 -> A == 0 and likewise for !=
// popcount(A) == bitwidth(A) -> A == -1 and likewise for != // popcount(A) == bitwidth(A) -> A == -1 and likewise for !=
bool IsZero = C.isNullValue(); bool IsZero = C.isZero();
if (IsZero || C == BitWidth) if (IsZero || C == BitWidth)
return new ICmpInst(Pred, II->getArgOperand(0), return new ICmpInst(Pred, II->getArgOperand(0),
IsZero ? Constant::getNullValue(Ty) IsZero ? Constant::getNullValue(Ty)
: Constant::getAllOnesValue(Ty)); : Constant::getAllOnesValue(Ty));
break; break;
} }
case Intrinsic::fshl: case Intrinsic::fshl:
case Intrinsic::fshr: case Intrinsic::fshr:
if (II->getArgOperand(0) == II->getArgOperand(1)) { if (II->getArgOperand(0) == II->getArgOperand(1)) {
// (rot X, ?) == 0/-1 --> X == 0/-1 // (rot X, ?) == 0/-1 --> X == 0/-1
// TODO: This transform is safe to re-use undef elts in a vector, but // TODO: This transform is safe to re-use undef elts in a vector, but
// the constant value passed in by the caller doesn't allow that. // the constant value passed in by the caller doesn't allow that.
if (C.isNullValue() || C.isAllOnesValue()) if (C.isZero() || C.isAllOnes())
return new ICmpInst(Pred, II->getArgOperand(0), Cmp.getOperand(1)); return new ICmpInst(Pred, II->getArgOperand(0), Cmp.getOperand(1));
const APInt *RotAmtC; const APInt *RotAmtC;
// ror(X, RotAmtC) == C --> X == rol(C, RotAmtC) // ror(X, RotAmtC) == C --> X == rol(C, RotAmtC)
// rol(X, RotAmtC) == C --> X == ror(C, RotAmtC) // rol(X, RotAmtC) == C --> X == ror(C, RotAmtC)
if (match(II->getArgOperand(2), m_APInt(RotAmtC))) if (match(II->getArgOperand(2), m_APInt(RotAmtC)))
return new ICmpInst(Pred, II->getArgOperand(0), return new ICmpInst(Pred, II->getArgOperand(0),
II->getIntrinsicID() == Intrinsic::fshl II->getIntrinsicID() == Intrinsic::fshl
? ConstantInt::get(Ty, C.rotr(*RotAmtC)) ? ConstantInt::get(Ty, C.rotr(*RotAmtC))
: ConstantInt::get(Ty, C.rotl(*RotAmtC))); : ConstantInt::get(Ty, C.rotl(*RotAmtC)));
} }
break; break;
case Intrinsic::uadd_sat: { case Intrinsic::uadd_sat: {
// uadd.sat(a, b) == 0 -> (a | b) == 0 // uadd.sat(a, b) == 0 -> (a | b) == 0
if (C.isNullValue()) { if (C.isZero()) {
Value *Or = Builder.CreateOr(II->getArgOperand(0), II->getArgOperand(1)); Value *Or = Builder.CreateOr(II->getArgOperand(0), II->getArgOperand(1));
return new ICmpInst(Pred, Or, Constant::getNullValue(Ty)); return new ICmpInst(Pred, Or, Constant::getNullValue(Ty));
} }
break; break;
} }
case Intrinsic::usub_sat: { case Intrinsic::usub_sat: {
// usub.sat(a, b) == 0 -> a <= b // usub.sat(a, b) == 0 -> a <= b
if (C.isNullValue()) { if (C.isZero()) {
ICmpInst::Predicate NewPred = ICmpInst::Predicate NewPred =
Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT;
return new ICmpInst(NewPred, II->getArgOperand(0), II->getArgOperand(1)); return new ICmpInst(NewPred, II->getArgOperand(0), II->getArgOperand(1));
} }
break; break;
} }
default: default:
break; break;
▲ Show 20 Lines • Show All 964 Lines • ▼ Show 20 Lines case Instruction::Xor: {
} }
break; break;
} }
case Instruction::Mul: { case Instruction::Mul: {
if (!I.isEquality()) if (!I.isEquality())
break; break;
const APInt *C; const APInt *C;
if (match(BO0->getOperand(1), m_APInt(C)) && !C->isNullValue() && if (match(BO0->getOperand(1), m_APInt(C)) && !C->isZero() &&
!C->isOneValue()) { !C->isOne()) {
// icmp eq/ne (X * C), (Y * C) --> icmp (X & Mask), (Y & Mask) // icmp eq/ne (X * C), (Y * C) --> icmp (X & Mask), (Y & Mask)
// Mask = -1 >> count-trailing-zeros(C). // Mask = -1 >> count-trailing-zeros(C).
if (unsigned TZs = C->countTrailingZeros()) { if (unsigned TZs = C->countTrailingZeros()) {
Constant *Mask = ConstantInt::get( Constant *Mask = ConstantInt::get(
BO0->getType(), BO0->getType(),
APInt::getLowBitsSet(C->getBitWidth(), C->getBitWidth() - TZs)); APInt::getLowBitsSet(C->getBitWidth(), C->getBitWidth() - TZs));
Value *And1 = Builder.CreateAnd(BO0->getOperand(0), Mask); Value *And1 = Builder.CreateAnd(BO0->getOperand(0), Mask);
Value *And2 = Builder.CreateAnd(BO1->getOperand(0), Mask); Value *And2 = Builder.CreateAnd(BO1->getOperand(0), Mask);
▲ Show 20 Lines • Show All 1,122 Lines • ▼ Show 20 Lines if (Op1Known.isZero()) {
auto NewPred = auto NewPred =
Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGE : CmpInst::ICMP_ULT; Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGE : CmpInst::ICMP_ULT;
return new ICmpInst(NewPred, X, CmpC); return new ICmpInst(NewPred, X, CmpC);
} }
} }
// Check if the LHS is 8 >>u x and the result is a power of 2 like 1. // Check if the LHS is 8 >>u x and the result is a power of 2 like 1.
const APInt *CI; const APInt *CI;
if (Op0KnownZeroInverted.isOneValue() && if (Op0KnownZeroInverted.isOne() &&
match(LHS, m_LShr(m_Power2(CI), m_Value(X)))) { match(LHS, m_LShr(m_Power2(CI), m_Value(X)))) {
// ((8 >>u X) & 1) == 0 -> X != 3 // ((8 >>u X) & 1) == 0 -> X != 3
// ((8 >>u X) & 1) != 0 -> X == 3 // ((8 >>u X) & 1) != 0 -> X == 3
unsigned CmpVal = CI->countTrailingZeros(); unsigned CmpVal = CI->countTrailingZeros();
auto NewPred = ICmpInst::getInversePredicate(Pred); auto NewPred = ICmpInst::getInversePredicate(Pred);
return new ICmpInst(NewPred, X, ConstantInt::get(X->getType(), CmpVal)); return new ICmpInst(NewPred, X, ConstantInt::get(X->getType(), CmpVal));
} }
} }
▲ Show 20 Lines • Show All 1,173 Lines • Show Last 20 Lines

llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp

Show First 20 Lines • Show All 710 Lines • ▼ Show 20 Lines
} }
/// True if C1 is a multiple of C2. Quotient contains C1/C2. /// True if C1 is a multiple of C2. Quotient contains C1/C2.
static bool isMultiple(const APInt &C1, const APInt &C2, APInt &Quotient, static bool isMultiple(const APInt &C1, const APInt &C2, APInt &Quotient,
bool IsSigned) { bool IsSigned) {
assert(C1.getBitWidth() == C2.getBitWidth() && "Constant widths not equal"); assert(C1.getBitWidth() == C2.getBitWidth() && "Constant widths not equal");
// Bail if we will divide by zero. // Bail if we will divide by zero.
if (C2.isNullValue()) if (C2.isZero())
return false; return false;
// Bail if we would divide INT_MIN by -1. // Bail if we would divide INT_MIN by -1.
if (IsSigned && C1.isMinSignedValue() && C2.isAllOnesValue()) if (IsSigned && C1.isMinSignedValue() && C2.isAllOnes())
return false; return false;
APInt Remainder(C1.getBitWidth(), /*val=*/0ULL, IsSigned); APInt Remainder(C1.getBitWidth(), /*val=*/0ULL, IsSigned);
if (IsSigned) if (IsSigned)
APInt::sdivrem(C1, C2, Quotient, Remainder); APInt::sdivrem(C1, C2, Quotient, Remainder);
else else
APInt::udivrem(C1, C2, Quotient, Remainder); APInt::udivrem(C1, C2, Quotient, Remainder);
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Lines if ((IsSigned && match(Op0, m_NSWShl(m_Value(X), m_APInt(C1))) &&
ConstantInt::get(Ty, Quotient)); ConstantInt::get(Ty, Quotient));
auto *OBO = cast<OverflowingBinaryOperator>(Op0); auto *OBO = cast<OverflowingBinaryOperator>(Op0);
Mul->setHasNoUnsignedWrap(!IsSigned && OBO->hasNoUnsignedWrap()); Mul->setHasNoUnsignedWrap(!IsSigned && OBO->hasNoUnsignedWrap());
Mul->setHasNoSignedWrap(OBO->hasNoSignedWrap()); Mul->setHasNoSignedWrap(OBO->hasNoSignedWrap());
return Mul; return Mul;
} }
} }
if (!C2->isNullValue()) // avoid X udiv 0 if (!C2->isZero()) // avoid X udiv 0
if (Instruction *FoldedDiv = foldBinOpIntoSelectOrPhi(I)) if (Instruction *FoldedDiv = foldBinOpIntoSelectOrPhi(I))
return FoldedDiv; return FoldedDiv;
} }
if (match(Op0, m_One())) { if (match(Op0, m_One())) {
assert(!Ty->isIntOrIntVectorTy(1) && "i1 divide not removed?"); assert(!Ty->isIntOrIntVectorTy(1) && "i1 divide not removed?");
if (IsSigned) { if (IsSigned) {
// If Op1 is 0 then it's undefined behaviour, if Op1 is 1 then the // If Op1 is 0 then it's undefined behaviour, if Op1 is 1 then the
▲ Show 20 Lines • Show All 794 Lines • Show Last 20 Lines

llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp

Show First 20 Lines • Show All 160 Lines • ▼ Show 20 Linesstatic Value *foldSelectICmpAnd(SelectInst &Sel, ICmpInst *Cmp,
} }
// In general, when both constants are non-zero, we would need an offset to // In general, when both constants are non-zero, we would need an offset to
// replace the select. This would require more instructions than we started // replace the select. This would require more instructions than we started
// with. But there's one special-case that we handle here because it can // with. But there's one special-case that we handle here because it can
// simplify/reduce the instructions. // simplify/reduce the instructions.
APInt TC = *SelTC; APInt TC = *SelTC;
APInt FC = *SelFC; APInt FC = *SelFC;
if (!TC.isNullValue() && !FC.isNullValue()) { if (!TC.isZero() && !FC.isZero()) {
// If the select constants differ by exactly one bit and that's the same // If the select constants differ by exactly one bit and that's the same
// bit that is masked and checked by the select condition, the select can // bit that is masked and checked by the select condition, the select can
// be replaced by bitwise logic to set/clear one bit of the constant result. // be replaced by bitwise logic to set/clear one bit of the constant result.
if (TC.getBitWidth() != AndMask.getBitWidth() || (TC ^ FC) != AndMask) if (TC.getBitWidth() != AndMask.getBitWidth() || (TC ^ FC) != AndMask)
return nullptr; return nullptr;
if (CreateAnd) { if (CreateAnd) {
// If we have to create an 'and', then we must kill the cmp to not // If we have to create an 'and', then we must kill the cmp to not
// increase the instruction count. // increase the instruction count.
Show All 20 Linesstatic Value *foldSelectICmpAnd(SelectInst &Sel, ICmpInst *Cmp,
} }
// Make sure one of the select arms is a power-of-2. // Make sure one of the select arms is a power-of-2.
if (!TC.isPowerOf2() && !FC.isPowerOf2()) if (!TC.isPowerOf2() && !FC.isPowerOf2())
return nullptr; return nullptr;
// Determine which shift is needed to transform result of the 'and' into the // Determine which shift is needed to transform result of the 'and' into the
// desired result. // desired result.
const APInt &ValC = !TC.isNullValue() ? TC : FC; const APInt &ValC = !TC.isZero() ? TC : FC;
unsigned ValZeros = ValC.logBase2(); unsigned ValZeros = ValC.logBase2();
unsigned AndZeros = AndMask.logBase2(); unsigned AndZeros = AndMask.logBase2();
// Insert the 'and' instruction on the input to the truncate. // Insert the 'and' instruction on the input to the truncate.
if (CreateAnd) if (CreateAnd)
V = Builder.CreateAnd(V, ConstantInt::get(V->getType(), AndMask)); V = Builder.CreateAnd(V, ConstantInt::get(V->getType(), AndMask));
// If types don't match, we can still convert the select by introducing a zext // If types don't match, we can still convert the select by introducing a zext
// or a trunc of the 'and'. // or a trunc of the 'and'.
if (ValZeros > AndZeros) { if (ValZeros > AndZeros) {
V = Builder.CreateZExtOrTrunc(V, SelType); V = Builder.CreateZExtOrTrunc(V, SelType);
V = Builder.CreateShl(V, ValZeros - AndZeros); V = Builder.CreateShl(V, ValZeros - AndZeros);
} else if (ValZeros < AndZeros) { } else if (ValZeros < AndZeros) {
V = Builder.CreateLShr(V, AndZeros - ValZeros); V = Builder.CreateLShr(V, AndZeros - ValZeros);
V = Builder.CreateZExtOrTrunc(V, SelType); V = Builder.CreateZExtOrTrunc(V, SelType);
} else { } else {
V = Builder.CreateZExtOrTrunc(V, SelType); V = Builder.CreateZExtOrTrunc(V, SelType);
} }
// Okay, now we know that everything is set up, we just don't know whether we // Okay, now we know that everything is set up, we just don't know whether we
// have a icmp_ne or icmp_eq and whether the true or false val is the zero. // have a icmp_ne or icmp_eq and whether the true or false val is the zero.
bool ShouldNotVal = !TC.isNullValue(); bool ShouldNotVal = !TC.isZero();
ShouldNotVal ^= Pred == ICmpInst::ICMP_NE; ShouldNotVal ^= Pred == ICmpInst::ICMP_NE;
if (ShouldNotVal) if (ShouldNotVal)
V = Builder.CreateXor(V, ValC); V = Builder.CreateXor(V, ValC);
return V; return V;
} }
/// We want to turn code that looks like this: /// We want to turn code that looks like this:
▲ Show 20 Lines • Show All 187 Lines • ▼ Show 20 Lines return TGEP->isInBounds() && FGEP->isInBounds()
? GetElementPtrInst::CreateInBounds(ElementType, Op0, {Op1}) ? GetElementPtrInst::CreateInBounds(ElementType, Op0, {Op1})
: GetElementPtrInst::Create(ElementType, Op0, {Op1}); : GetElementPtrInst::Create(ElementType, Op0, {Op1});
} }
llvm_unreachable("Expected BinaryOperator or GEP"); llvm_unreachable("Expected BinaryOperator or GEP");
return nullptr; return nullptr;
} }
static bool isSelect01(const APInt &C1I, const APInt &C2I) { static bool isSelect01(const APInt &C1I, const APInt &C2I) {
if (!C1I.isNullValue() && !C2I.isNullValue()) // One side must be zero. if (!C1I.isZero() && !C2I.isZero()) // One side must be zero.
return false; return false;
return C1I.isOneValue() || C1I.isAllOnesValue() || return C1I.isOne() || C1I.isAllOnes() || C2I.isOne() || C2I.isAllOnes();
C2I.isOneValue() || C2I.isAllOnesValue();
} }
/// Try to fold the select into one of the operands to allow further /// Try to fold the select into one of the operands to allow further
/// optimization. /// optimization.
Instruction *InstCombinerImpl::foldSelectIntoOp(SelectInst &SI, Value *TrueVal, Instruction *InstCombinerImpl::foldSelectIntoOp(SelectInst &SI, Value *TrueVal,
Value *FalseVal) { Value *FalseVal) {
// See the comment above GetSelectFoldableOperands for a description of the // See the comment above GetSelectFoldableOperands for a description of the
// transformation we are doing here. // transformation we are doing here.
▲ Show 20 Lines • Show All 1,428 Lines • ▼ Show 20 Lines auto IsSignedSaturateLimit = [&](Value *Limit, bool IsAdd) {
ICmpInst::Predicate Pred; ICmpInst::Predicate Pred;
Value *TrueVal, *FalseVal, *Op; Value *TrueVal, *FalseVal, *Op;
const APInt *C; const APInt *C;
if (!match(Limit, m_Select(m_ICmp(Pred, m_Value(Op), m_APInt(C)), if (!match(Limit, m_Select(m_ICmp(Pred, m_Value(Op), m_APInt(C)),
m_Value(TrueVal), m_Value(FalseVal)))) m_Value(TrueVal), m_Value(FalseVal))))
return false; return false;
auto IsZeroOrOne = [](const APInt &C) { auto IsZeroOrOne = [](const APInt &C) { return C.isZero() || C.isOne(); };
return C.isNullValue() || C.isOneValue();
};
auto IsMinMax = [&](Value *Min, Value *Max) { auto IsMinMax = [&](Value *Min, Value *Max) {
APInt MinVal = APInt::getSignedMinValue(Ty->getScalarSizeInBits()); APInt MinVal = APInt::getSignedMinValue(Ty->getScalarSizeInBits());
APInt MaxVal = APInt::getSignedMaxValue(Ty->getScalarSizeInBits()); APInt MaxVal = APInt::getSignedMaxValue(Ty->getScalarSizeInBits());
return match(Min, m_SpecificInt(MinVal)) && return match(Min, m_SpecificInt(MinVal)) &&
match(Max, m_SpecificInt(MaxVal)); match(Max, m_SpecificInt(MaxVal));
}; };
if (Op != X && Op != Y) if (Op != X && Op != Y)
▲ Show 20 Lines • Show All 1,359 Lines • ▼ Show 20 LinesInstruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
// If we can compute the condition, there's no need for a select. // If we can compute the condition, there's no need for a select.
// Like the above fold, we are attempting to reduce compile-time cost by // Like the above fold, we are attempting to reduce compile-time cost by
// putting this fold here with limitations rather than in InstSimplify. // putting this fold here with limitations rather than in InstSimplify.
// The motivation for this call into value tracking is to take advantage of // The motivation for this call into value tracking is to take advantage of
// the assumption cache, so make sure that is populated. // the assumption cache, so make sure that is populated.
if (!CondVal->getType()->isVectorTy() && !AC.assumptions().empty()) { if (!CondVal->getType()->isVectorTy() && !AC.assumptions().empty()) {
KnownBits Known(1); KnownBits Known(1);
computeKnownBits(CondVal, Known, 0, &SI); computeKnownBits(CondVal, Known, 0, &SI);
if (Known.One.isOneValue()) if (Known.One.isOne())
return replaceInstUsesWith(SI, TrueVal); return replaceInstUsesWith(SI, TrueVal);
if (Known.Zero.isOneValue()) if (Known.Zero.isOne())
return replaceInstUsesWith(SI, FalseVal); return replaceInstUsesWith(SI, FalseVal);
} }
if (Instruction *BitCastSel = foldSelectCmpBitcasts(SI, Builder)) if (Instruction *BitCastSel = foldSelectCmpBitcasts(SI, Builder))
return BitCastSel; return BitCastSel;
// Simplify selects that test the returned flag of cmpxchg instructions. // Simplify selects that test the returned flag of cmpxchg instructions.
if (Value *V = foldSelectCmpXchg(SI)) if (Value *V = foldSelectCmpXchg(SI))
▲ Show 20 Lines • Show All 51 Lines • Show Last 20 Lines

llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp

Show First 20 Lines • Show All 118 Lines • ▼ Show 20 Lines assert(
"Value *V, DemandedMask and Known must have same BitWidth"); "Value *V, DemandedMask and Known must have same BitWidth");
if (isa<Constant>(V)) { if (isa<Constant>(V)) {
computeKnownBits(V, Known, Depth, CxtI); computeKnownBits(V, Known, Depth, CxtI);
return nullptr; return nullptr;
} }
Known.resetAll(); Known.resetAll();
if (DemandedMask.isNullValue()) // Not demanding any bits from V. if (DemandedMask.isZero()) // Not demanding any bits from V.
return UndefValue::get(VTy); return UndefValue::get(VTy);
if (Depth == MaxAnalysisRecursionDepth) if (Depth == MaxAnalysisRecursionDepth)
return nullptr; return nullptr;
if (isa<ScalableVectorType>(VTy)) if (isa<ScalableVectorType>(VTy))
return nullptr; return nullptr;
▲ Show 20 Lines • Show All 133 Lines • ▼ Show 20 Lines if (DemandedMask.isSubsetOf(RHSKnown.Zero|RHSKnown.One) &&
Instruction *And = BinaryOperator::CreateAnd(I->getOperand(0), AndC); Instruction *And = BinaryOperator::CreateAnd(I->getOperand(0), AndC);
return InsertNewInstWith(And, *I); return InsertNewInstWith(And, *I);
} }
// If the RHS is a constant, see if we can change it. Don't alter a -1 // If the RHS is a constant, see if we can change it. Don't alter a -1
// constant because that's a canonical 'not' op, and that is better for // constant because that's a canonical 'not' op, and that is better for
// combining, SCEV, and codegen. // combining, SCEV, and codegen.
const APInt *C; const APInt *C;
if (match(I->getOperand(1), m_APInt(C)) && !C->isAllOnesValue()) { if (match(I->getOperand(1), m_APInt(C)) && !C->isAllOnes()) {
if ((*C | ~DemandedMask).isAllOnesValue()) { if ((*C | ~DemandedMask).isAllOnes()) {
// Force bits to 1 to create a 'not' op. // Force bits to 1 to create a 'not' op.
I->setOperand(1, ConstantInt::getAllOnesValue(VTy)); I->setOperand(1, ConstantInt::getAllOnesValue(VTy));
return I; return I;
} }
// If we can't turn this into a 'not', try to shrink the constant. // If we can't turn this into a 'not', try to shrink the constant.
if (ShrinkDemandedConstant(I, 1, DemandedMask)) if (ShrinkDemandedConstant(I, 1, DemandedMask))
return I; return I;
} }
▲ Show 20 Lines • Show All 242 Lines • ▼ Show 20 Lines case Instruction::Sub: {
} }
// If we are known to be adding/subtracting zeros to every bit below // If we are known to be adding/subtracting zeros to every bit below
// the highest demanded bit, we just return the other side. // the highest demanded bit, we just return the other side.
if (DemandedFromOps.isSubsetOf(RHSKnown.Zero)) if (DemandedFromOps.isSubsetOf(RHSKnown.Zero))
return I->getOperand(0); return I->getOperand(0);
// We can't do this with the LHS for subtraction, unless we are only // We can't do this with the LHS for subtraction, unless we are only
// demanding the LSB. // demanding the LSB.
if ((I->getOpcode() == Instruction::Add || if ((I->getOpcode() == Instruction::Add || DemandedFromOps.isOne()) &&
DemandedFromOps.isOneValue()) &&
DemandedFromOps.isSubsetOf(LHSKnown.Zero)) DemandedFromOps.isSubsetOf(LHSKnown.Zero))
return I->getOperand(1); return I->getOperand(1);
// Otherwise just compute the known bits of the result. // Otherwise just compute the known bits of the result.
bool NSW = cast<OverflowingBinaryOperator>(I)->hasNoSignedWrap(); bool NSW = cast<OverflowingBinaryOperator>(I)->hasNoSignedWrap();
Known = KnownBits::computeForAddSub(I->getOpcode() == Instruction::Add, Known = KnownBits::computeForAddSub(I->getOpcode() == Instruction::Add,
NSW, LHSKnown, RHSKnown); NSW, LHSKnown, RHSKnown);
break; break;
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines case Instruction::LShr: {
} }
break; break;
} }
case Instruction::AShr: { case Instruction::AShr: {
// If this is an arithmetic shift right and only the low-bit is set, we can // If this is an arithmetic shift right and only the low-bit is set, we can
// always convert this into a logical shr, even if the shift amount is // always convert this into a logical shr, even if the shift amount is
// variable. The low bit of the shift cannot be an input sign bit unless // variable. The low bit of the shift cannot be an input sign bit unless
// the shift amount is >= the size of the datatype, which is undefined. // the shift amount is >= the size of the datatype, which is undefined.
if (DemandedMask.isOneValue()) { if (DemandedMask.isOne()) {
// Perform the logical shift right. // Perform the logical shift right.
Instruction *NewVal = BinaryOperator::CreateLShr( Instruction *NewVal = BinaryOperator::CreateLShr(
I->getOperand(0), I->getOperand(1), I->getName()); I->getOperand(0), I->getOperand(1), I->getName());
return InsertNewInstWith(NewVal, *I); return InsertNewInstWith(NewVal, *I);
} }
// If the sign bit is the only bit demanded by this ashr, then there is no // If the sign bit is the only bit demanded by this ashr, then there is no
// need to do it, the shift doesn't change the high bit. // need to do it, the shift doesn't change the high bit.
▲ Show 20 Lines • Show All 488 Lines • ▼ Show 20 LinesValue *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V,
assert((DemandedElts & ~EltMask) == 0 && "Invalid DemandedElts!"); assert((DemandedElts & ~EltMask) == 0 && "Invalid DemandedElts!");
if (match(V, m_Undef())) { if (match(V, m_Undef())) {
// If the entire vector is undef or poison, just return this info. // If the entire vector is undef or poison, just return this info.
UndefElts = EltMask; UndefElts = EltMask;
return nullptr; return nullptr;
} }
if (DemandedElts.isNullValue()) { // If nothing is demanded, provide poison. if (DemandedElts.isZero()) { // If nothing is demanded, provide poison.
UndefElts = EltMask; UndefElts = EltMask;
return PoisonValue::get(V->getType()); return PoisonValue::get(V->getType());
} }
UndefElts = 0; UndefElts = 0;
if (auto *C = dyn_cast<Constant>(V)) { if (auto *C = dyn_cast<Constant>(V)) {
// Check if this is identity. If so, return 0 since we are not simplifying // Check if this is identity. If so, return 0 since we are not simplifying
// anything. // anything.
if (DemandedElts.isAllOnesValue()) if (DemandedElts.isAllOnes())
return nullptr; return nullptr;
Type *EltTy = cast<VectorType>(V->getType())->getElementType(); Type *EltTy = cast<VectorType>(V->getType())->getElementType();
Constant *Poison = PoisonValue::get(EltTy); Constant *Poison = PoisonValue::get(EltTy);
SmallVector<Constant*, 16> Elts; SmallVector<Constant*, 16> Elts;
for (unsigned i = 0; i != VWidth; ++i) { for (unsigned i = 0; i != VWidth; ++i) {
if (!DemandedElts[i]) { // If not demanded, set to poison. if (!DemandedElts[i]) { // If not demanded, set to poison.
Elts.push_back(Poison); Elts.push_back(Poison);
▲ Show 20 Lines • Show All 136 Lines • ▼ Show 20 Lines case Instruction::ShuffleVector: {
assert(Shuffle->getOperand(0)->getType() == assert(Shuffle->getOperand(0)->getType() ==
Shuffle->getOperand(1)->getType() && Shuffle->getOperand(1)->getType() &&
"Expected shuffle operands to have same type"); "Expected shuffle operands to have same type");
unsigned OpWidth = cast<FixedVectorType>(Shuffle->getOperand(0)->getType()) unsigned OpWidth = cast<FixedVectorType>(Shuffle->getOperand(0)->getType())
->getNumElements(); ->getNumElements();
// Handle trivial case of a splat. Only check the first element of LHS // Handle trivial case of a splat. Only check the first element of LHS
// operand. // operand.
if (all_of(Shuffle->getShuffleMask(), [](int Elt) { return Elt == 0; }) && if (all_of(Shuffle->getShuffleMask(), [](int Elt) { return Elt == 0; }) &&
DemandedElts.isAllOnesValue()) { DemandedElts.isAllOnes()) {
if (!match(I->getOperand(1), m_Undef())) { if (!match(I->getOperand(1), m_Undef())) {
I->setOperand(1, PoisonValue::get(I->getOperand(1)->getType())); I->setOperand(1, PoisonValue::get(I->getOperand(1)->getType()));
MadeChange = true; MadeChange = true;
} }
APInt LeftDemanded(OpWidth, 1); APInt LeftDemanded(OpWidth, 1);
APInt LHSUndefElts(OpWidth, 0); APInt LHSUndefElts(OpWidth, 0);
simplifyAndSetOp(I, 0, LeftDemanded, LHSUndefElts); simplifyAndSetOp(I, 0, LeftDemanded, LHSUndefElts);
if (LHSUndefElts[0]) if (LHSUndefElts[0])
▲ Show 20 Lines • Show All 291 Lines • ▼ Show 20 Lines if (match(I, m_BinOp(BO)) && !BO->isIntDivRem() && !BO->isShift()) {
// Output elements are undefined if both are undefined. Consider things // Output elements are undefined if both are undefined. Consider things
// like undef & 0. The result is known zero, not undef. // like undef & 0. The result is known zero, not undef.
UndefElts &= UndefElts2; UndefElts &= UndefElts2;
} }
// If we've proven all of the lanes undef, return an undef value. // If we've proven all of the lanes undef, return an undef value.
// TODO: Intersect w/demanded lanes // TODO: Intersect w/demanded lanes
if (UndefElts.isAllOnesValue()) if (UndefElts.isAllOnes())
return UndefValue::get(I->getType());; return UndefValue::get(I->getType());;
return MadeChange ? I : nullptr; return MadeChange ? I : nullptr;
} }

llvm/lib/Transforms/InstCombine/InstructionCombining.cpp

Show First 20 Lines • Show All 1,053 Lines • ▼ Show 20 Lines if (CI->hasOneUse()) {
// TODO: Deal with FP constants? // TODO: Deal with FP constants?
if (!A->getType()->isIntOrIntVectorTy() || A->getType() != B->getType()) if (!A->getType()->isIntOrIntVectorTy() || A->getType() != B->getType())
return false; return false;
// Compare for equality including undefs as equal. // Compare for equality including undefs as equal.
auto *Cmp = ConstantExpr::getCompare(ICmpInst::ICMP_EQ, ConstA, ConstB); auto *Cmp = ConstantExpr::getCompare(ICmpInst::ICMP_EQ, ConstA, ConstB);
const APInt *C; const APInt *C;
return match(Cmp, m_APIntAllowUndef(C)) && C->isOneValue(); return match(Cmp, m_APIntAllowUndef(C)) && C->isOne();
}; };
if ((areLooselyEqual(TV, Op0) && areLooselyEqual(FV, Op1)) || if ((areLooselyEqual(TV, Op0) && areLooselyEqual(FV, Op1)) ||
(areLooselyEqual(FV, Op0) && areLooselyEqual(TV, Op1))) (areLooselyEqual(FV, Op0) && areLooselyEqual(TV, Op1)))
return nullptr; return nullptr;
} }
} }
▲ Show 20 Lines • Show All 3,211 Lines • Show Last 20 Lines

llvm/lib/Transforms/Scalar/BDCE.cpp

Show First 20 Lines • Show All 47 Lines • ▼ Show 20 Linesstatic void clearAssumptionsOfUsers(Instruction *I, DemandedBits &DB) {
// Initialize the worklist with eligible direct users. // Initialize the worklist with eligible direct users.
SmallPtrSet<Instruction *, 16> Visited; SmallPtrSet<Instruction *, 16> Visited;
SmallVector<Instruction *, 16> WorkList; SmallVector<Instruction *, 16> WorkList;
for (User *JU : I->users()) { for (User *JU : I->users()) {
// If all bits of a user are demanded, then we know that nothing below that // If all bits of a user are demanded, then we know that nothing below that
// in the def-use chain needs to be changed. // in the def-use chain needs to be changed.
auto *J = dyn_cast<Instruction>(JU); auto *J = dyn_cast<Instruction>(JU);
if (J && J->getType()->isIntOrIntVectorTy() && if (J && J->getType()->isIntOrIntVectorTy() &&
!DB.getDemandedBits(J).isAllOnesValue()) { !DB.getDemandedBits(J).isAllOnes()) {
Visited.insert(J); Visited.insert(J);
WorkList.push_back(J); WorkList.push_back(J);
} }
// Note that we need to check for non-int types above before asking for // Note that we need to check for non-int types above before asking for
// demanded bits. Normally, the only way to reach an instruction with an // demanded bits. Normally, the only way to reach an instruction with an
// non-int type is via an instruction that has side effects (or otherwise // non-int type is via an instruction that has side effects (or otherwise
// will demand its input bits). However, if we have a readnone function // will demand its input bits). However, if we have a readnone function
Show All 14 Lines while (!WorkList.empty()) {
// 1. llvm.assume demands its operand, so trivializing can't change it. // 1. llvm.assume demands its operand, so trivializing can't change it.
// 2. range metadata only applies to memory accesses which demand all bits. // 2. range metadata only applies to memory accesses which demand all bits.
for (User *KU : J->users()) { for (User *KU : J->users()) {
// If all bits of a user are demanded, then we know that nothing below // If all bits of a user are demanded, then we know that nothing below
// that in the def-use chain needs to be changed. // that in the def-use chain needs to be changed.
auto *K = dyn_cast<Instruction>(KU); auto *K = dyn_cast<Instruction>(KU);
if (K && Visited.insert(K).second && K->getType()->isIntOrIntVectorTy() && if (K && Visited.insert(K).second && K->getType()->isIntOrIntVectorTy() &&
!DB.getDemandedBits(K).isAllOnesValue()) !DB.getDemandedBits(K).isAllOnes())
WorkList.push_back(K); WorkList.push_back(K);
} }
} }
} }
static bool bitTrackingDCE(Function &F, DemandedBits &DB) { static bool bitTrackingDCE(Function &F, DemandedBits &DB) {
SmallVector<Instruction*, 128> Worklist; SmallVector<Instruction*, 128> Worklist;
bool Changed = false; bool Changed = false;
for (Instruction &I : instructions(F)) { for (Instruction &I : instructions(F)) {
// If the instruction has side effects and no non-dbg uses, // If the instruction has side effects and no non-dbg uses,
// skip it. This way we avoid computing known bits on an instruction // skip it. This way we avoid computing known bits on an instruction
// that will not help us. // that will not help us.
if (I.mayHaveSideEffects() && I.use_empty()) if (I.mayHaveSideEffects() && I.use_empty())
continue; continue;
// Remove instructions that are dead, either because they were not reached // Remove instructions that are dead, either because they were not reached
// during analysis or have no demanded bits. // during analysis or have no demanded bits.
if (DB.isInstructionDead(&I) || if (DB.isInstructionDead(&I) ||
(I.getType()->isIntOrIntVectorTy() && (I.getType()->isIntOrIntVectorTy() && DB.getDemandedBits(&I).isZero() &&
DB.getDemandedBits(&I).isNullValue() &&
wouldInstructionBeTriviallyDead(&I))) { wouldInstructionBeTriviallyDead(&I))) {
Worklist.push_back(&I); Worklist.push_back(&I);
Changed = true; Changed = true;
continue; continue;
} }
// Convert SExt into ZExt if none of the extension bits is required // Convert SExt into ZExt if none of the extension bits is required
if (SExtInst *SE = dyn_cast<SExtInst>(&I)) { if (SExtInst *SE = dyn_cast<SExtInst>(&I)) {
▲ Show 20 Lines • Show All 93 Lines • Show Last 20 Lines

llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp

Show First 20 Lines • Show All 683 Lines • ▼ Show 20 Lines if (LHS == RHS)
return SE.getConstant(LHS->getType(), 1); return SE.getConstant(LHS->getType(), 1);
// Handle a few RHS special cases. // Handle a few RHS special cases.
const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS); const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS);
if (RC) { if (RC) {
const APInt &RA = RC->getAPInt(); const APInt &RA = RC->getAPInt();
// Handle x /s -1 as x * -1, to give ScalarEvolution a chance to do // Handle x /s -1 as x * -1, to give ScalarEvolution a chance to do
// some folding. // some folding.
if (RA.isAllOnesValue()) { if (RA.isAllOnes()) {
if (LHS->getType()->isPointerTy()) if (LHS->getType()->isPointerTy())
return nullptr; return nullptr;
return SE.getMulExpr(LHS, RC); return SE.getMulExpr(LHS, RC);
} }
// Handle x /s 1 as x. // Handle x /s 1 as x.
if (RA == 1) if (RA == 1)
return LHS; return LHS;
} }
▲ Show 20 Lines • Show All 5,715 Lines • Show Last 20 Lines

llvm/lib/Transforms/Scalar/Reassociate.cpp

Show First 20 Lines • Show All 1,273 Lines • ▼ Show 20 Lines
/// Helper function of CombineXorOpnd(). It creates a bitwise-and /// Helper function of CombineXorOpnd(). It creates a bitwise-and
/// instruction with the given two operands, and return the resulting /// instruction with the given two operands, and return the resulting
/// instruction. There are two special cases: 1) if the constant operand is 0, /// instruction. There are two special cases: 1) if the constant operand is 0,
/// it will return NULL. 2) if the constant is ~0, the symbolic operand will /// it will return NULL. 2) if the constant is ~0, the symbolic operand will
/// be returned. /// be returned.
static Value *createAndInstr(Instruction *InsertBefore, Value *Opnd, static Value *createAndInstr(Instruction *InsertBefore, Value *Opnd,
const APInt &ConstOpnd) { const APInt &ConstOpnd) {
if (ConstOpnd.isNullValue()) if (ConstOpnd.isZero())
return nullptr; return nullptr;
if (ConstOpnd.isAllOnesValue()) if (ConstOpnd.isAllOnes())
return Opnd; return Opnd;
Instruction *I = BinaryOperator::CreateAnd( Instruction *I = BinaryOperator::CreateAnd(
Opnd, ConstantInt::get(Opnd->getType(), ConstOpnd), "and.ra", Opnd, ConstantInt::get(Opnd->getType(), ConstOpnd), "and.ra",
InsertBefore); InsertBefore);
I->setDebugLoc(InsertBefore->getDebugLoc()); I->setDebugLoc(InsertBefore->getDebugLoc());
return I; return I;
} }
// Helper function of OptimizeXor(). It tries to simplify "Opnd1 ^ ConstOpnd" // Helper function of OptimizeXor(). It tries to simplify "Opnd1 ^ ConstOpnd"
// into "R ^ C", where C would be 0, and R is a symbolic value. // into "R ^ C", where C would be 0, and R is a symbolic value.
// //
// If it was successful, true is returned, and the "R" and "C" is returned // If it was successful, true is returned, and the "R" and "C" is returned
// via "Res" and "ConstOpnd", respectively; otherwise, false is returned, // via "Res" and "ConstOpnd", respectively; otherwise, false is returned,
// and both "Res" and "ConstOpnd" remain unchanged. // and both "Res" and "ConstOpnd" remain unchanged.
bool ReassociatePass::CombineXorOpnd(Instruction *I, XorOpnd *Opnd1, bool ReassociatePass::CombineXorOpnd(Instruction *I, XorOpnd *Opnd1,
APInt &ConstOpnd, Value *&Res) { APInt &ConstOpnd, Value *&Res) {
// Xor-Rule 1: (x | c1) ^ c2 = (x | c1) ^ (c1 ^ c1) ^ c2 // Xor-Rule 1: (x | c1) ^ c2 = (x | c1) ^ (c1 ^ c1) ^ c2
// = ((x | c1) ^ c1) ^ (c1 ^ c2) // = ((x | c1) ^ c1) ^ (c1 ^ c2)
// = (x & ~c1) ^ (c1 ^ c2) // = (x & ~c1) ^ (c1 ^ c2)
// It is useful only when c1 == c2. // It is useful only when c1 == c2.
if (!Opnd1->isOrExpr() || Opnd1->getConstPart().isNullValue()) if (!Opnd1->isOrExpr() || Opnd1->getConstPart().isZero())
return false; return false;
if (!Opnd1->getValue()->hasOneUse()) if (!Opnd1->getValue()->hasOneUse())
return false; return false;
const APInt &C1 = Opnd1->getConstPart(); const APInt &C1 = Opnd1->getConstPart();
if (C1 != ConstOpnd) if (C1 != ConstOpnd)
return false; return false;
▲ Show 20 Lines • Show All 147 Lines • ▼ Show 20 LinesValue *ReassociatePass::OptimizeXor(Instruction *I,
XorOpnd *PrevOpnd = nullptr; XorOpnd *PrevOpnd = nullptr;
bool Changed = false; bool Changed = false;
for (unsigned i = 0, e = Opnds.size(); i < e; i++) { for (unsigned i = 0, e = Opnds.size(); i < e; i++) {
XorOpnd *CurrOpnd = OpndPtrs[i]; XorOpnd *CurrOpnd = OpndPtrs[i];
// The combined value // The combined value
Value *CV; Value *CV;
// Step 3.1: Try simplifying "CurrOpnd ^ ConstOpnd" // Step 3.1: Try simplifying "CurrOpnd ^ ConstOpnd"
if (!ConstOpnd.isNullValue() && if (!ConstOpnd.isZero() && CombineXorOpnd(I, CurrOpnd, ConstOpnd, CV)) {
CombineXorOpnd(I, CurrOpnd, ConstOpnd, CV)) {
Changed = true; Changed = true;
if (CV) if (CV)
*CurrOpnd = XorOpnd(CV); *CurrOpnd = XorOpnd(CV);
else { else {
CurrOpnd->Invalidate(); CurrOpnd->Invalidate();
continue; continue;
} }
} }
Show All 24 Lines if (Changed) {
Ops.clear(); Ops.clear();
for (unsigned int i = 0, e = Opnds.size(); i < e; i++) { for (unsigned int i = 0, e = Opnds.size(); i < e; i++) {
XorOpnd &O = Opnds[i]; XorOpnd &O = Opnds[i];
if (O.isInvalid()) if (O.isInvalid())
continue; continue;
ValueEntry VE(getRank(O.getValue()), O.getValue()); ValueEntry VE(getRank(O.getValue()), O.getValue());
Ops.push_back(VE); Ops.push_back(VE);
} }
if (!ConstOpnd.isNullValue()) { if (!ConstOpnd.isZero()) {
Value *C = ConstantInt::get(Ty, ConstOpnd); Value *C = ConstantInt::get(Ty, ConstOpnd);
ValueEntry VE(getRank(C), C); ValueEntry VE(getRank(C), C);
Ops.push_back(VE); Ops.push_back(VE);
} }
unsigned Sz = Ops.size(); unsigned Sz = Ops.size();
if (Sz == 1) if (Sz == 1)
return Ops.back().Op; return Ops.back().Op;
if (Sz == 0) { if (Sz == 0) {
assert(ConstOpnd.isNullValue()); assert(ConstOpnd.isZero());
return ConstantInt::get(Ty, ConstOpnd); return ConstantInt::get(Ty, ConstOpnd);
} }
} }
return nullptr; return nullptr;
} }
/// Optimize a series of operands to an 'add' instruction. This /// Optimize a series of operands to an 'add' instruction. This
▲ Show 20 Lines • Show All 1,086 Lines • Show Last 20 Lines

llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp

Show First 20 Lines • Show All 601 Lines • ▼ Show 20 Lines else
BumpWithUglyGEP = true; BumpWithUglyGEP = true;
} }
// Compute Bump = C - Basis = (i' - i) * S. // Compute Bump = C - Basis = (i' - i) * S.
// Common case 1: if (i' - i) is 1, Bump = S. // Common case 1: if (i' - i) is 1, Bump = S.
if (IndexOffset == 1) if (IndexOffset == 1)
return C.Stride; return C.Stride;
// Common case 2: if (i' - i) is -1, Bump = -S. // Common case 2: if (i' - i) is -1, Bump = -S.
if (IndexOffset.isAllOnesValue()) if (IndexOffset.isAllOnes())
return Builder.CreateNeg(C.Stride); return Builder.CreateNeg(C.Stride);
// Otherwise, Bump = (i' - i) * sext/trunc(S). Note that (i' - i) and S may // Otherwise, Bump = (i' - i) * sext/trunc(S). Note that (i' - i) and S may
// have different bit widths. // have different bit widths.
IntegerType *DeltaType = IntegerType *DeltaType =
IntegerType::get(Basis.Ins->getContext(), IndexOffset.getBitWidth()); IntegerType::get(Basis.Ins->getContext(), IndexOffset.getBitWidth());
Value *ExtendedStride = Builder.CreateSExtOrTrunc(C.Stride, DeltaType); Value *ExtendedStride = Builder.CreateSExtOrTrunc(C.Stride, DeltaType);
if (IndexOffset.isPowerOf2()) { if (IndexOffset.isPowerOf2()) {
▲ Show 20 Lines • Show All 161 Lines • Show Last 20 Lines

llvm/lib/Transforms/Utils/Local.cpp

Show First 20 Lines • Show All 3,193 Lines • ▼ Show 20 Lines auto *Trunc =
CastInst::CreateIntegerCast(Provider, DemandedTy, false, "trunc", I); CastInst::CreateIntegerCast(Provider, DemandedTy, false, "trunc", I);
InsertedInsts.push_back(Trunc); InsertedInsts.push_back(Trunc);
Provider = Trunc; Provider = Trunc;
} }
Instruction *Result = CallInst::Create(F, Provider, "rev", I); Instruction *Result = CallInst::Create(F, Provider, "rev", I);
InsertedInsts.push_back(Result); InsertedInsts.push_back(Result);
if (!DemandedMask.isAllOnesValue()) { if (!DemandedMask.isAllOnes()) {
auto *Mask = ConstantInt::get(DemandedTy, DemandedMask); auto *Mask = ConstantInt::get(DemandedTy, DemandedMask);
Result = BinaryOperator::Create(Instruction::And, Result, Mask, "mask", I); Result = BinaryOperator::Create(Instruction::And, Result, Mask, "mask", I);
InsertedInsts.push_back(Result); InsertedInsts.push_back(Result);
} }
// We may need to zeroextend back to the result type. // We may need to zeroextend back to the result type.
if (ITy != Result->getType()) { if (ITy != Result->getType()) {
auto *ExtInst = CastInst::CreateIntegerCast(Result, ITy, false, "zext", I); auto *ExtInst = CastInst::CreateIntegerCast(Result, ITy, false, "zext", I);
▲ Show 20 Lines • Show All 157 Lines • Show Last 20 Lines

llvm/unittests/IR/ConstantRangeTest.cpp

Show First 20 Lines • Show All 637 Lines • ▼ Show 20 Lines EnumerateConstantRanges(Bits, [&](const ConstantRange &CR) {
EXPECT_EQ(Exact, ResultCR); EXPECT_EQ(Exact, ResultCR);
}); });
} }
TEST_F(ConstantRangeTest, losslessUnsignedTruncationZeroext) { TEST_F(ConstantRangeTest, losslessUnsignedTruncationZeroext) {
unsigned Bits = 4; unsigned Bits = 4;
EnumerateConstantRanges(Bits, [&](const ConstantRange &CR) { EnumerateConstantRanges(Bits, [&](const ConstantRange &CR) {
unsigned MinBitWidth = CR.getActiveBits(); unsigned MinBitWidth = CR.getActiveBits();
if (MinBitWidth == 0) { if (MinBitWidth == 0) {
EXPECT_TRUE(CR.isEmptySet() || (CR.isSingleElement() && EXPECT_TRUE(CR.isEmptySet() ||
CR.getSingleElement()->isNullValue())); (CR.isSingleElement() && CR.getSingleElement()->isZero()));
return; return;
} }
if (MinBitWidth == Bits) if (MinBitWidth == Bits)
return; return;
EXPECT_EQ(CR, CR.truncate(MinBitWidth).zeroExtend(Bits)); EXPECT_EQ(CR, CR.truncate(MinBitWidth).zeroExtend(Bits));
}); });
} }
▲ Show 20 Lines • Show All 564 Lines • ▼ Show 20 Lines EnumerateTwoConstantRanges(Bits, [&](const ConstantRange &CR1,
BitVector Results(1 << Bits); BitVector Results(1 << Bits);
ForeachNumInConstantRange(CR1, [&](const APInt &N1) { ForeachNumInConstantRange(CR1, [&](const APInt &N1) {
ForeachNumInConstantRange(CR2, [&](const APInt &N2) { ForeachNumInConstantRange(CR2, [&](const APInt &N2) {
// Division by zero is UB. // Division by zero is UB.
if (N2 == 0) if (N2 == 0)
return; return;
// SignedMin / -1 is UB. // SignedMin / -1 is UB.
if (N1.isMinSignedValue() && N2.isAllOnesValue()) if (N1.isMinSignedValue() && N2.isAllOnes())
return; return;
APInt N = N1.sdiv(N2); APInt N = N1.sdiv(N2);
Results.set(N.getSExtValue() + Bias); Results.set(N.getSExtValue() + Bias);
}); });
}); });
ConstantRange CR = CR1.sdiv(CR2); ConstantRange CR = CR1.sdiv(CR2);
▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines EXPECT_EQ(ConstantRange(APInt(16, 12), APInt(16, 15))
.urem(ConstantRange(APInt(16, 10))), .urem(ConstantRange(APInt(16, 10))),
ConstantRange(APInt(16, 0), APInt(16, 10))); ConstantRange(APInt(16, 0), APInt(16, 10)));
TestBinaryOpExhaustiveCorrectnessOnly( TestBinaryOpExhaustiveCorrectnessOnly(
[](const ConstantRange &CR1, const ConstantRange &CR2) { [](const ConstantRange &CR1, const ConstantRange &CR2) {
return CR1.urem(CR2); return CR1.urem(CR2);
}, },
[](const APInt &N1, const APInt &N2) -> Optional<APInt> { [](const APInt &N1, const APInt &N2) -> Optional<APInt> {
if (N2.isNullValue()) if (N2.isZero())
return None; return None;
return N1.urem(N2); return N1.urem(N2);
}); });
} }
TEST_F(ConstantRangeTest, SRem) { TEST_F(ConstantRangeTest, SRem) {
EXPECT_EQ(Full.srem(Empty), Empty); EXPECT_EQ(Full.srem(Empty), Empty);
EXPECT_EQ(Empty.srem(Full), Empty); EXPECT_EQ(Empty.srem(Full), Empty);
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines EXPECT_EQ(ConstantRange(APInt(16, 12), APInt(16, 15))
.srem(ConstantRange(APInt(16, 10))), .srem(ConstantRange(APInt(16, 10))),
ConstantRange(APInt(16, 0), APInt(16, 10))); ConstantRange(APInt(16, 0), APInt(16, 10)));
TestBinaryOpExhaustiveCorrectnessOnly( TestBinaryOpExhaustiveCorrectnessOnly(
[](const ConstantRange &CR1, const ConstantRange &CR2) { [](const ConstantRange &CR1, const ConstantRange &CR2) {
return CR1.srem(CR2); return CR1.srem(CR2);
}, },
[](const APInt &N1, const APInt &N2) -> Optional<APInt> { [](const APInt &N1, const APInt &N2) -> Optional<APInt> {
if (N2.isNullValue()) if (N2.isZero())
return None; return None;
return N1.srem(N2); return N1.srem(N2);
}); });
} }
TEST_F(ConstantRangeTest, Shl) { TEST_F(ConstantRangeTest, Shl) {
ConstantRange Some2(APInt(16, 0xfff), APInt(16, 0x8000)); ConstantRange Some2(APInt(16, 0xfff), APInt(16, 0x8000));
ConstantRange WrapNullMax(APInt(16, 0x1), APInt(16, 0x0)); ConstantRange WrapNullMax(APInt(16, 0x1), APInt(16, 0x0));
▲ Show 20 Lines • Show All 1,104 Lines • Show Last 20 Lines

llvm/unittests/IR/PatternMatch.cpp

Show First 20 Lines • Show All 1,086 Lines • ▼ Show 20 LinesTEST_F(PatternMatchTest, VectorUndefInt) {
EXPECT_FALSE(match(ScalarUndef, m_APIntAllowUndef(C))); EXPECT_FALSE(match(ScalarUndef, m_APIntAllowUndef(C)));
EXPECT_FALSE(match(VectorUndef, m_APInt(C))); EXPECT_FALSE(match(VectorUndef, m_APInt(C)));
EXPECT_FALSE(match(VectorUndef, m_APIntForbidUndef(C))); EXPECT_FALSE(match(VectorUndef, m_APIntForbidUndef(C)));
EXPECT_FALSE(match(VectorUndef, m_APIntAllowUndef(C))); EXPECT_FALSE(match(VectorUndef, m_APIntAllowUndef(C)));
// We can always match simple constants and simple splats. // We can always match simple constants and simple splats.
C = nullptr; C = nullptr;
EXPECT_TRUE(match(ScalarZero, m_APInt(C))); EXPECT_TRUE(match(ScalarZero, m_APInt(C)));
EXPECT_TRUE(C->isNullValue()); EXPECT_TRUE(C->isZero());
C = nullptr; C = nullptr;
EXPECT_TRUE(match(ScalarZero, m_APIntForbidUndef(C))); EXPECT_TRUE(match(ScalarZero, m_APIntForbidUndef(C)));
EXPECT_TRUE(C->isNullValue()); EXPECT_TRUE(C->isZero());
C = nullptr; C = nullptr;
EXPECT_TRUE(match(ScalarZero, m_APIntAllowUndef(C))); EXPECT_TRUE(match(ScalarZero, m_APIntAllowUndef(C)));
EXPECT_TRUE(C->isNullValue()); EXPECT_TRUE(C->isZero());
C = nullptr; C = nullptr;
EXPECT_TRUE(match(VectorZero, m_APInt(C))); EXPECT_TRUE(match(VectorZero, m_APInt(C)));
EXPECT_TRUE(C->isNullValue()); EXPECT_TRUE(C->isZero());
C = nullptr; C = nullptr;
EXPECT_TRUE(match(VectorZero, m_APIntForbidUndef(C))); EXPECT_TRUE(match(VectorZero, m_APIntForbidUndef(C)));
EXPECT_TRUE(C->isNullValue()); EXPECT_TRUE(C->isZero());
C = nullptr; C = nullptr;
EXPECT_TRUE(match(VectorZero, m_APIntAllowUndef(C))); EXPECT_TRUE(match(VectorZero, m_APIntAllowUndef(C)));
EXPECT_TRUE(C->isNullValue()); EXPECT_TRUE(C->isZero());
// Whether splats with undef can be matched depends on the matcher. // Whether splats with undef can be matched depends on the matcher.
EXPECT_FALSE(match(VectorZeroUndef, m_APInt(C))); EXPECT_FALSE(match(VectorZeroUndef, m_APInt(C)));
EXPECT_FALSE(match(VectorZeroUndef, m_APIntForbidUndef(C))); EXPECT_FALSE(match(VectorZeroUndef, m_APIntForbidUndef(C)));
C = nullptr; C = nullptr;
EXPECT_TRUE(match(VectorZeroUndef, m_APIntAllowUndef(C))); EXPECT_TRUE(match(VectorZeroUndef, m_APIntAllowUndef(C)));
EXPECT_TRUE(C->isNullValue()); EXPECT_TRUE(C->isZero());
} }
TEST_F(PatternMatchTest, VectorUndefFloat) { TEST_F(PatternMatchTest, VectorUndefFloat) {
Type *ScalarTy = IRB.getFloatTy(); Type *ScalarTy = IRB.getFloatTy();
Type *VectorTy = FixedVectorType::get(ScalarTy, 4); Type *VectorTy = FixedVectorType::get(ScalarTy, 4);
Constant *ScalarUndef = UndefValue::get(ScalarTy); Constant *ScalarUndef = UndefValue::get(ScalarTy);
Constant *VectorUndef = UndefValue::get(VectorTy); Constant *VectorUndef = UndefValue::get(VectorTy);
Constant *ScalarZero = Constant::getNullValue(ScalarTy); Constant *ScalarZero = Constant::getNullValue(ScalarTy);
▲ Show 20 Lines • Show All 309 Lines • ▼ Show 20 Lines EXPECT_FALSE(
match(Intrinsic5, m_Intrinsic<Intrinsic::instrprof_increment_step>( match(Intrinsic5, m_Intrinsic<Intrinsic::instrprof_increment_step>(
m_Value(), m_Value(), m_Value(), m_Value(), m_Value(), m_Value(), m_Value(), m_Value(),
m_SpecificInt(10)))); m_SpecificInt(10))));
} }
namespace { namespace {
struct is_unsigned_zero_pred { struct is_unsigned_zero_pred {
bool isValue(const APInt &C) { return C.isNullValue(); } bool isValue(const APInt &C) { return C.isZero(); }
}; };
struct is_float_zero_pred { struct is_float_zero_pred {
bool isValue(const APFloat &C) { return C.isZero(); } bool isValue(const APFloat &C) { return C.isZero(); }
}; };
template <typename T> struct always_true_pred { template <typename T> struct always_true_pred {
bool isValue(const T &) { return true; } bool isValue(const T &) { return true; }
▲ Show 20 Lines • Show All 234 Lines • Show Last 20 Lines

llvm/unittests/Support/KnownBitsTest.cpp

Show First 20 Lines • Show All 160 Lines • ▼ Show 20 Lines ForeachKnownBits(Bits, [&](const KnownBits &Known2) {
Res = (N1.sext(2 * Bits) * N2.sext(2 * Bits)).extractBits(Bits, Bits); Res = (N1.sext(2 * Bits) * N2.sext(2 * Bits)).extractBits(Bits, Bits);
KnownMulHS.One &= Res; KnownMulHS.One &= Res;
KnownMulHS.Zero &= ~Res; KnownMulHS.Zero &= ~Res;
Res = (N1.zext(2 * Bits) * N2.zext(2 * Bits)).extractBits(Bits, Bits); Res = (N1.zext(2 * Bits) * N2.zext(2 * Bits)).extractBits(Bits, Bits);
KnownMulHU.One &= Res; KnownMulHU.One &= Res;
KnownMulHU.Zero &= ~Res; KnownMulHU.Zero &= ~Res;
if (!N2.isNullValue()) { if (!N2.isZero()) {
Res = N1.udiv(N2); Res = N1.udiv(N2);
KnownUDiv.One &= Res; KnownUDiv.One &= Res;
KnownUDiv.Zero &= ~Res; KnownUDiv.Zero &= ~Res;
Res = N1.urem(N2); Res = N1.urem(N2);
KnownURem.One &= Res; KnownURem.One &= Res;
KnownURem.Zero &= ~Res; KnownURem.Zero &= ~Res;
▲ Show 20 Lines • Show All 322 Lines • Show Last 20 Lines

llvm/utils/TableGen/PredicateExpander.cpp

Show First 20 Lines • Show All 464 Lines • ▼ Show 20 Lines for (unsigned I = 0, E = ProcModelMask.getActiveBits(); I < E; ++I) {
FirstProcID = false; FirstProcID = false;
} }
OS << ") {\n"; OS << ") {\n";
increaseIndentLevel(); increaseIndentLevel();
OS.indent(getIndentLevel() * 2); OS.indent(getIndentLevel() * 2);
if (ShouldUpdateOpcodeMask) { if (ShouldUpdateOpcodeMask) {
if (PI.OperandMask.isNullValue()) if (PI.OperandMask.isZero())
OS << "Mask.clearAllBits();\n"; OS << "Mask.clearAllBits();\n";
else else
OS << "Mask = " << PI.OperandMask << ";\n"; OS << "Mask = " << PI.OperandMask << ";\n";
OS.indent(getIndentLevel() * 2); OS.indent(getIndentLevel() * 2);
} }
OS << "return "; OS << "return ";
expandPredicate(OS, PI.Predicate); expandPredicate(OS, PI.Predicate);
OS << ";\n"; OS << ";\n";
▲ Show 20 Lines • Show All 68 Lines • Show Last 20 Lines