diff --git a/llvm/include/llvm/Analysis/Loads.h b/llvm/include/llvm/Analysis/Loads.h --- a/llvm/include/llvm/Analysis/Loads.h +++ b/llvm/include/llvm/Analysis/Loads.h @@ -74,6 +74,24 @@ Instruction *ScanFrom = nullptr, const DominatorTree *DT = nullptr); +/// Returns the number of bytes known to be dereferenceable for the +/// pointer value \p V. +/// +/// If \p CanBeNull is set by this function the pointer can be null. +/// If \p IsKnownDeref is set by this function the pointer is known to be +/// dereferenceable up to the returned number of bytes. +/// +/// Note that both \p CanBeNull and \p IsKnownDeref can be set if null is a +/// valid pointer. +uint64_t getPointerDereferenceableBytes(const Value *V, const DataLayout &DL, + bool &CanBeNull, bool &IsKnownDeref); + +/// Returns an alignment of the pointer value \p V. +/// +/// Returns an alignment which is either specified explicitly, e.g. via +/// align attribute of a function argument, or guaranteed by DataLayout. +unsigned getPointerAlignment(const Value *V, const DataLayout &DL); + /// The default number of maximum instructions to scan in the block, used by /// FindAvailableLoadedValue(). extern cl::opt DefMaxInstsToScan; diff --git a/llvm/include/llvm/IR/Value.h b/llvm/include/llvm/IR/Value.h --- a/llvm/include/llvm/IR/Value.h +++ b/llvm/include/llvm/IR/Value.h @@ -620,20 +620,6 @@ static_cast(this)->stripInBoundsOffsets()); } - /// Returns the number of bytes known to be dereferenceable for the - /// pointer value. - /// - /// If CanBeNull is set by this function the pointer can either be null or be - /// dereferenceable up to the returned number of bytes. - uint64_t getPointerDereferenceableBytes(const DataLayout &DL, - bool &CanBeNull) const; - - /// Returns an alignment of the pointer value. - /// - /// Returns an alignment which is either specified explicitly, e.g. via - /// align attribute of a function argument, or guaranteed by DataLayout. - unsigned getPointerAlignment(const DataLayout &DL) const; - /// Translate PHI node to its predecessor from the given basic block. /// /// If this value is a PHI node with CurBB as its parent, return the value in diff --git a/llvm/lib/Analysis/CaptureTracking.cpp b/llvm/lib/Analysis/CaptureTracking.cpp --- a/llvm/lib/Analysis/CaptureTracking.cpp +++ b/llvm/lib/Analysis/CaptureTracking.cpp @@ -20,6 +20,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/CFG.h" +#include "llvm/Analysis/Loads.h" #include "llvm/Analysis/OrderedBasicBlock.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Constants.h" @@ -45,8 +46,8 @@ if (auto *GEP = dyn_cast(O)) if (GEP->isInBounds()) return true; - bool CanBeNull; - return O->getPointerDereferenceableBytes(DL, CanBeNull); + bool CanBeNull, IsKnownDeref; + return getPointerDereferenceableBytes(O, DL, CanBeNull, IsKnownDeref); } namespace { diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp --- a/llvm/lib/Analysis/Loads.cpp +++ b/llvm/lib/Analysis/Loads.cpp @@ -22,21 +22,15 @@ #include "llvm/IR/Operator.h" #include "llvm/IR/Statepoint.h" +#include + using namespace llvm; static bool isAligned(const Value *Base, const APInt &Offset, unsigned Align, const DataLayout &DL) { - APInt BaseAlign(Offset.getBitWidth(), Base->getPointerAlignment(DL)); - - if (!BaseAlign) { - Type *Ty = Base->getType()->getPointerElementType(); - if (!Ty->isSized()) - return false; - BaseAlign = DL.getABITypeAlignment(Ty); - } - + APInt BaseAlign(Offset.getBitWidth(), + std::max(1U, getPointerAlignment(Base, DL))); APInt Alignment(Offset.getBitWidth(), Align); - assert(Alignment.isPowerOf2() && "must be a power of 2!"); return BaseAlign.uge(Alignment) && !(Offset & (Alignment-1)); } @@ -48,71 +42,268 @@ return isAligned(Base, Offset, Align, DL); } -/// Test if V is always a pointer to allocated and suitably aligned memory for -/// a simple load or store. -static bool isDereferenceableAndAlignedPointer( - const Value *V, unsigned Align, const APInt &Size, const DataLayout &DL, - const Instruction *CtxI, const DominatorTree *DT, - SmallPtrSetImpl &Visited) { - // Already visited? Bail out, we've likely hit unreachable code. +static unsigned getPointerAlignment(const Value *V, const DataLayout &DL, + SmallPtrSetImpl &Visited) { + assert(V->getType()->isPointerTy() && "must be pointer"); + + static const unsigned MAX_ALIGN = 1U << 29; + // The visited set catches recursion for "invalid" SSA instructions and allows + // recursion on PHI nodes (not yet done). if (!Visited.insert(V).second) - return false; + return MAX_ALIGN; + + const Value *Stripped = V->stripPointerCastsSameRepresentation(); + if (Stripped != V) + return getPointerAlignment(Stripped, DL, Visited); + + unsigned Align = 0; + if (auto *GO = dyn_cast(V)) { + if (isa(GO)) { + MaybeAlign FunctionPtrAlign = DL.getFunctionPtrAlign(); + unsigned Align = FunctionPtrAlign ? FunctionPtrAlign->value() : 0; + switch (DL.getFunctionPtrAlignType()) { + case DataLayout::FunctionPtrAlignType::Independent: + return Align; + case DataLayout::FunctionPtrAlignType::MultipleOfFunctionAlign: + return std::max(Align, GO->getAlignment()); + } + } + Align = GO->getAlignment(); + if (Align == 0) { + if (auto *GVar = dyn_cast(GO)) { + Type *ObjectType = GVar->getValueType(); + if (ObjectType->isSized()) { + // If the object is defined in the current Module, we'll be giving + // it the preferred alignment. Otherwise, we have to assume that it + // may only have the minimum ABI alignment. + if (GVar->isStrongDefinitionForLinker()) + Align = DL.getPreferredAlignment(GVar); + else + Align = DL.getABITypeAlignment(ObjectType); + } + } + } + } else if (auto *A = dyn_cast(V)) { + Align = A->getParamAlignment(); + } else if (auto *AI = dyn_cast(V)) { + Align = AI->getAlignment(); + if (Align == 0) { + Type *AllocatedType = AI->getAllocatedType(); + if (AllocatedType->isSized()) + Align = DL.getPrefTypeAlignment(AllocatedType); + } + } else if (auto *RI = dyn_cast(V)) { + // For gc.relocate, look through relocations + Align = getPointerAlignment(RI->getDerivedPtr(), DL, Visited); + } else if (auto *Call = dyn_cast(V)) { + Align = Call->getRetAlignment(); + if (Align == 0 && Call->getCalledFunction()) + Align = Call->getCalledFunction()->getAttributes().getRetAlignment(); + if (Align == 0) + if (auto *RP = getArgumentAliasingToReturnedPointer(Call, true)) + Align = getPointerAlignment(RP, DL, Visited); + } else if (auto *LI = dyn_cast(V)) { + if (MDNode *MD = LI->getMetadata(LLVMContext::MD_align)) { + ConstantInt *CI = mdconst::extract(MD->getOperand(0)); + Align = CI->getLimitedValue(); + } + } else if (auto *ASC = dyn_cast(V)) { + Align = getPointerAlignment(ASC->getPointerOperand(), DL, Visited); + } else if (auto *GEP = dyn_cast(V)) { + const Value *Base = GEP->getPointerOperand(); - // Note that it is not safe to speculate into a malloc'd region because - // malloc may return null. + APInt Offset(DL.getIndexTypeSizeInBits(GEP->getType()), 0); + if (GEP->accumulateConstantOffset(DL, Offset) && Offset != 0) { + auto BaseAlign = getPointerAlignment(Base, DL, Visited); + if (BaseAlign <= 1) { + // Propagate a base alignment of 1 and give up on a zero alignment. + Align = BaseAlign; + } else { + // For the purpose of alignment computation we treat negative offsets as + // if they were positive, so we only discuss positive offsets below. + // We assume k_x values are existentially quantified. + // + // Base has alignment BA, thus: + // Base = k_0 * BA + // GEP equals Base + Offset, thus: + // GEP = k_0 * BA + Offset + // With GCD = gcd(BA, Offset), BA = k_1 * GCD, and Offset = k_2 * GCD we + // can express GEP as follows: + // GEP = k_0 * (k_1 * GCD) + k_2 * GCD + // The common factor in both terms is GCD, so we can express it as: + // GEP = GCD * (k_0 * k_1 + k_2) + // Which implies that the GEP has GCD alignment. + APInt GCD = APIntOps::GreatestCommonDivisor( + Offset.abs(), APInt(Offset.getBitWidth(), BaseAlign)); + Align = GCD.getZExtValue(); - // bitcast instructions are no-ops as far as dereferenceability is concerned. - if (const BitCastOperator *BC = dyn_cast(V)) - return isDereferenceableAndAlignedPointer(BC->getOperand(0), Align, Size, - DL, CtxI, DT, Visited); + } + } + // Do not use type information below to improve the alignment. + return Align; + } - bool CheckForNonNull = false; - APInt KnownDerefBytes(Size.getBitWidth(), - V->getPointerDereferenceableBytes(DL, CheckForNonNull)); - if (KnownDerefBytes.getBoolValue()) { - if (KnownDerefBytes.uge(Size)) - if (!CheckForNonNull || isKnownNonZero(V, DL, 0, nullptr, CtxI, DT)) - return isAligned(V, Align, DL); + if (!Align) { + Type *Ty = V->getType()->getPointerElementType(); + if (Ty->isSized()) + Align = DL.getABITypeAlignment(Ty); } - // For GEPs, determine if the indexing lands within the allocated object. - if (const GEPOperator *GEP = dyn_cast(V)) { - const Value *Base = GEP->getPointerOperand(); + return Align; +} - APInt Offset(DL.getIndexTypeSizeInBits(GEP->getType()), 0); - if (!GEP->accumulateConstantOffset(DL, Offset) || Offset.isNegative() || - !Offset.urem(APInt(Offset.getBitWidth(), Align)).isMinValue()) - return false; +unsigned llvm::getPointerAlignment(const Value *V, const DataLayout &DL) { + SmallPtrSet Visited; + return ::getPointerAlignment(V, DL, Visited); +} - // If the base pointer is dereferenceable for Offset+Size bytes, then the - // GEP (== Base + Offset) is dereferenceable for Size bytes. If the base - // pointer is aligned to Align bytes, and the Offset is divisible by Align - // then the GEP (== Base + Offset == k_0 * Align + k_1 * Align) is also - // aligned to Align bytes. - - // Offset and Size may have different bit widths if we have visited an - // addrspacecast, so we can't do arithmetic directly on the APInt values. - return isDereferenceableAndAlignedPointer( - Base, Align, Offset + Size.sextOrTrunc(Offset.getBitWidth()), - DL, CtxI, DT, Visited); - } +static uint64_t +getPointerDereferenceableBytes(const Value *V, const DataLayout &DL, + bool &CanBeNull, bool &IsKnownDeref, + SmallPtrSetImpl &Visited) { + assert(V->getType()->isPointerTy() && "must be pointer"); + CanBeNull = false; + IsKnownDeref = true; - // For gc.relocate, look through relocations - if (const GCRelocateInst *RelocateInst = dyn_cast(V)) - return isDereferenceableAndAlignedPointer( - RelocateInst->getDerivedPtr(), Align, Size, DL, CtxI, DT, Visited); + // The visited set catches recursion for "invalid" SSA instructions and allows + // recursion on PHI nodes (not yet done). + if (!Visited.insert(V).second) + return std::numeric_limits::max(); + + const Value *Stripped = V->stripPointerCastsSameRepresentation(); + if (Stripped != V) + return getPointerDereferenceableBytes(Stripped, DL, CanBeNull, IsKnownDeref, + Visited); + + const Function *F = nullptr; + uint64_t DerefBytes = 0; + CanBeNull = false; + if (auto *A = dyn_cast(V)) { + F = A->getParent(); + DerefBytes = A->getDereferenceableBytes(); + if (DerefBytes == 0 && (A->hasByValAttr() || A->hasStructRetAttr())) { + Type *PT = cast(A->getType())->getElementType(); + if (PT->isSized()) + DerefBytes = DL.getTypeStoreSize(PT); + } + if (DerefBytes == 0) { + DerefBytes = A->getDereferenceableOrNullBytes(); + CanBeNull = true; + } + } else if (auto *LI = dyn_cast(V)) { + if (MDNode *MD = LI->getMetadata(LLVMContext::MD_dereferenceable)) { + ConstantInt *CI = mdconst::extract(MD->getOperand(0)); + DerefBytes = CI->getLimitedValue(); + } + if (DerefBytes == 0) { + if (MDNode *MD = + LI->getMetadata(LLVMContext::MD_dereferenceable_or_null)) { + ConstantInt *CI = mdconst::extract(MD->getOperand(0)); + DerefBytes = CI->getLimitedValue(); + } + CanBeNull = true; + } + } else if (auto *IP = dyn_cast(V)) { + if (MDNode *MD = IP->getMetadata(LLVMContext::MD_dereferenceable)) { + ConstantInt *CI = mdconst::extract(MD->getOperand(0)); + DerefBytes = CI->getLimitedValue(); + } + if (DerefBytes == 0) { + if (MDNode *MD = + IP->getMetadata(LLVMContext::MD_dereferenceable_or_null)) { + ConstantInt *CI = mdconst::extract(MD->getOperand(0)); + DerefBytes = CI->getLimitedValue(); + } + CanBeNull = true; + } + } else if (auto *AI = dyn_cast(V)) { + if (!AI->isArrayAllocation()) { + DerefBytes = DL.getTypeStoreSize(AI->getAllocatedType()); + CanBeNull = false; + } + } else if (auto *GV = dyn_cast(V)) { + if (GV->getValueType()->isSized() && !GV->hasExternalWeakLinkage()) { + // TODO: Don't outright reject hasExternalWeakLinkage but set the + // CanBeNull flag. + DerefBytes = DL.getTypeStoreSize(GV->getValueType()); + CanBeNull = false; + } + } else if (auto *RI = dyn_cast(V)) { + // For gc.relocate, look through relocations, must be checkd before CallBase. + return getPointerDereferenceableBytes(RI->getDerivedPtr(), DL, CanBeNull, + IsKnownDeref, Visited); + } else if (auto *ASC = dyn_cast(V)) { + DerefBytes = getPointerDereferenceableBytes( + ASC->getPointerOperand(), DL, CanBeNull, IsKnownDeref, Visited); + } else if (auto *GEP = dyn_cast(V)) { + APInt Offset(DL.getIndexTypeSizeInBits(GEP->getType()), 0); + // Give up on non-constant GEPs. + if (!GEP->accumulateConstantOffset(DL, Offset)) + return 0; + + // If Base has N dereferenceable bytes and is O (=Offset) away from the + // GEP, then: + // - if 0 is null, we take the base result. + // - if O is positive, the GEP has max(0, N - O) dereferenceable bytes + // because it is O bytes advanced from the base. + // - if O is negative and inbounds, the GEP has N + abs(O) dereferenceable + // bytes because inbounds need to stay in + // the same allocation. + // - if O is negative and not inbounds, the GEP has 0 dereferenceable bytes + // because we do not know if we are + // still in the allocation or not. + // + // The "can be null" is set to false if we have an offset that is not null + // and an inbounds GEP or dereferenceable bytes. Note that this is later + // overwritten if null pointers are defined. + const Value *Base = GEP->getPointerOperand(); + uint64_t BaseDerefBytes = getPointerDereferenceableBytes( + Base, DL, CanBeNull, IsKnownDeref, Visited); + if (Offset == 0) { + DerefBytes = BaseDerefBytes; + } else if (Offset.getSExtValue() > 0) { + DerefBytes = + std::max(int64_t(BaseDerefBytes - Offset.getZExtValue()), int64_t(0)); + CanBeNull = !GEP->isInBounds(); + } else { + assert(Offset.getSExtValue() < 0 && "Did not expect zero offset!"); + if (GEP->isInBounds()) + DerefBytes = BaseDerefBytes + Offset.abs().getZExtValue(); + else + DerefBytes = 0; + CanBeNull = !GEP->isInBounds(); + } + // Assume dereferenceable implies nonnull here but note that we later revert + // the decision if it does not based on the pointer address space. + CanBeNull &= (DerefBytes == 0); + } else if (const auto *Call = dyn_cast(V)) { + DerefBytes = Call->getDereferenceableBytes(AttributeList::ReturnIndex); + if (DerefBytes == 0) { + DerefBytes = + Call->getDereferenceableOrNullBytes(AttributeList::ReturnIndex); + CanBeNull = true; + } + if (DerefBytes == 0) + if (auto *RP = getArgumentAliasingToReturnedPointer(Call, true)) + DerefBytes = getPointerDereferenceableBytes(RP, DL, CanBeNull, + IsKnownDeref, Visited); + } - if (const AddrSpaceCastInst *ASC = dyn_cast(V)) - return isDereferenceableAndAlignedPointer(ASC->getOperand(0), Align, Size, - DL, CtxI, DT, Visited); + if (auto *I = dyn_cast(V)) + F = I->getFunction(); - if (const auto *Call = dyn_cast(V)) - if (auto *RP = getArgumentAliasingToReturnedPointer(Call, true)) - return isDereferenceableAndAlignedPointer(RP, Align, Size, DL, CtxI, DT, - Visited); + IsKnownDeref = !CanBeNull; + CanBeNull |= NullPointerIsDefined(F, V->getType()->getPointerAddressSpace()); + return DerefBytes; +} - // If we don't know, assume the worst. - return false; +uint64_t llvm::getPointerDereferenceableBytes(const Value *V, + const DataLayout &DL, + bool &CanBeNull, + bool &IsKnownDeref) { + SmallPtrSet Visited; + return ::getPointerDereferenceableBytes(V, DL, CanBeNull, IsKnownDeref, + Visited); } bool llvm::isDereferenceableAndAlignedPointer(const Value *V, unsigned Align, @@ -120,9 +311,17 @@ const DataLayout &DL, const Instruction *CtxI, const DominatorTree *DT) { - SmallPtrSet Visited; - return ::isDereferenceableAndAlignedPointer(V, Align, Size, DL, CtxI, DT, - Visited); + bool IsKnownDeref = false, CanBeNull = false; + APInt KnownDerefBytes( + Size.getBitWidth(), + getPointerDereferenceableBytes(V, DL, CanBeNull, IsKnownDeref)); + if (KnownDerefBytes.getBoolValue()) + if (KnownDerefBytes.uge(Size)) + if (IsKnownDeref || isKnownNonZero(V, DL, 0, nullptr, CtxI, DT)) + return ::isAligned(V, Align, DL); + + // If we don't know, assume the worst. + return false; } bool llvm::isDereferenceableAndAlignedPointer(const Value *V, Type *Ty, @@ -142,11 +341,10 @@ if (!Ty->isSized()) return false; - SmallPtrSet Visited; return ::isDereferenceableAndAlignedPointer( V, Align, APInt(DL.getIndexTypeSizeInBits(V->getType()), DL.getTypeStoreSize(Ty)), - DL, CtxI, DT, Visited); + DL, CtxI, DT); } bool llvm::isDereferenceablePointer(const Value *V, Type *Ty, diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -1721,7 +1721,7 @@ // Aligned pointers have trailing zeros - refine Known.Zero set if (V->getType()->isPointerTy()) { - unsigned Align = V->getPointerAlignment(Q.DL); + unsigned Align = getPointerAlignment(V, Q.DL); if (Align) Known.Zero.setLowBits(countTrailingZeros(Align)); } diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -25,6 +25,7 @@ #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/ConstantFolding.h" #include "llvm/Analysis/InstructionSimplify.h" +#include "llvm/Analysis/Loads.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/MemoryBuiltins.h" #include "llvm/Analysis/ProfileSummaryInfo.h" @@ -1819,7 +1820,7 @@ // forbidden. GlobalVariable *GV; if ((GV = dyn_cast(Val)) && GV->canIncreaseAlignment() && - GV->getPointerAlignment(*DL) < PrefAlign && + getPointerAlignment(GV, *DL) < PrefAlign && DL->getTypeAllocSize(GV->getValueType()) >= MinSize + Offset2) GV->setAlignment(PrefAlign); @@ -1857,7 +1858,7 @@ case Intrinsic::experimental_widenable_condition: { // Give up on future widening oppurtunties so that we can fold away dead // paths and merge blocks before going into block-local instruction - // selection. + // selection. if (II->use_empty()) { II->eraseFromParent(); return true; diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp --- a/llvm/lib/IR/ConstantFold.cpp +++ b/llvm/lib/IR/ConstantFold.cpp @@ -19,6 +19,7 @@ #include "ConstantFold.h" #include "llvm/ADT/APSInt.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/Loads.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" @@ -1125,10 +1126,38 @@ isa(CE1->getOperand(0))) { GlobalValue *GV = cast(CE1->getOperand(0)); - unsigned GVAlign; + unsigned GVAlign = 0; if (Module *TheModule = GV->getParent()) { - GVAlign = GV->getPointerAlignment(TheModule->getDataLayout()); + const DataLayout &DL = TheModule->getDataLayout(); + // Deduction of alignment is done via getPointerAlignment but that + // function is not accessible from the IR library so we copied the + // logic here. + if (isa(GV)) { + MaybeAlign FunctionPtrAlign = DL.getFunctionPtrAlign(); + GVAlign = FunctionPtrAlign ? FunctionPtrAlign->value() : 0; + switch (DL.getFunctionPtrAlignType()) { + case DataLayout::FunctionPtrAlignType::Independent: + break; + case DataLayout::FunctionPtrAlignType::MultipleOfFunctionAlign: + GVAlign = std::max(GVAlign, GV->getAlignment()); + } + } + if (GVAlign == 0) { + GVAlign = GV->getAlignment(); + if (auto *GVar = dyn_cast(GV)) { + Type *ObjectType = GVar->getValueType(); + if (ObjectType->isSized()) { + // If the object is defined in the current Module, we'll be + // giving it the preferred alignment. Otherwise, we have to + // assume that it may only have the minimum ABI alignment. + if (GVar->isStrongDefinitionForLinker()) + GVAlign = DL.getPreferredAlignment(GVar); + else + GVAlign = DL.getABITypeAlignment(ObjectType); + } + } + } // If the function alignment is not specified then assume that it // is 4. diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp --- a/llvm/lib/IR/Value.cpp +++ b/llvm/lib/IR/Value.cpp @@ -15,6 +15,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SetVector.h" +#include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" @@ -609,139 +610,6 @@ return stripPointerCastsAndOffsets(this); } -uint64_t Value::getPointerDereferenceableBytes(const DataLayout &DL, - bool &CanBeNull) const { - assert(getType()->isPointerTy() && "must be pointer"); - - auto AS = getType()->getPointerAddressSpace(); - bool NullIsValid = (AS != 0); - if (auto *I = dyn_cast(this)) - NullIsValid = NullPointerIsDefined(I->getFunction(), AS); - - uint64_t DerefBytes = 0; - CanBeNull = false; - if (const Argument *A = dyn_cast(this)) { - NullIsValid = NullPointerIsDefined(A->getParent(), AS); - DerefBytes = A->getDereferenceableBytes(); - if (DerefBytes == 0 && (A->hasByValAttr() || A->hasStructRetAttr())) { - Type *PT = cast(A->getType())->getElementType(); - if (PT->isSized()) - DerefBytes = DL.getTypeStoreSize(PT); - } - if (DerefBytes == 0) { - DerefBytes = A->getDereferenceableOrNullBytes(); - CanBeNull = true; - } - } else if (const auto *Call = dyn_cast(this)) { - DerefBytes = Call->getDereferenceableBytes(AttributeList::ReturnIndex); - if (DerefBytes == 0) { - DerefBytes = - Call->getDereferenceableOrNullBytes(AttributeList::ReturnIndex); - CanBeNull = true; - } - } else if (const LoadInst *LI = dyn_cast(this)) { - if (MDNode *MD = LI->getMetadata(LLVMContext::MD_dereferenceable)) { - ConstantInt *CI = mdconst::extract(MD->getOperand(0)); - DerefBytes = CI->getLimitedValue(); - } - if (DerefBytes == 0) { - if (MDNode *MD = - LI->getMetadata(LLVMContext::MD_dereferenceable_or_null)) { - ConstantInt *CI = mdconst::extract(MD->getOperand(0)); - DerefBytes = CI->getLimitedValue(); - } - CanBeNull = true; - } - } else if (auto *IP = dyn_cast(this)) { - if (MDNode *MD = IP->getMetadata(LLVMContext::MD_dereferenceable)) { - ConstantInt *CI = mdconst::extract(MD->getOperand(0)); - DerefBytes = CI->getLimitedValue(); - } - if (DerefBytes == 0) { - if (MDNode *MD = - IP->getMetadata(LLVMContext::MD_dereferenceable_or_null)) { - ConstantInt *CI = mdconst::extract(MD->getOperand(0)); - DerefBytes = CI->getLimitedValue(); - } - CanBeNull = true; - } - } else if (auto *AI = dyn_cast(this)) { - if (!AI->isArrayAllocation()) { - DerefBytes = DL.getTypeStoreSize(AI->getAllocatedType()); - CanBeNull = false; - } - } else if (auto *GV = dyn_cast(this)) { - if (GV->getValueType()->isSized() && !GV->hasExternalWeakLinkage()) { - // TODO: Don't outright reject hasExternalWeakLinkage but set the - // CanBeNull flag. - DerefBytes = DL.getTypeStoreSize(GV->getValueType()); - CanBeNull = false; - } - } - - CanBeNull = !NullIsValid; - return DerefBytes; -} - -unsigned Value::getPointerAlignment(const DataLayout &DL) const { - assert(getType()->isPointerTy() && "must be pointer"); - - unsigned Align = 0; - if (auto *GO = dyn_cast(this)) { - if (isa(GO)) { - MaybeAlign FunctionPtrAlign = DL.getFunctionPtrAlign(); - unsigned Align = FunctionPtrAlign ? FunctionPtrAlign->value() : 0; - switch (DL.getFunctionPtrAlignType()) { - case DataLayout::FunctionPtrAlignType::Independent: - return Align; - case DataLayout::FunctionPtrAlignType::MultipleOfFunctionAlign: - return std::max(Align, GO->getAlignment()); - } - } - Align = GO->getAlignment(); - if (Align == 0) { - if (auto *GVar = dyn_cast(GO)) { - Type *ObjectType = GVar->getValueType(); - if (ObjectType->isSized()) { - // If the object is defined in the current Module, we'll be giving - // it the preferred alignment. Otherwise, we have to assume that it - // may only have the minimum ABI alignment. - if (GVar->isStrongDefinitionForLinker()) - Align = DL.getPreferredAlignment(GVar); - else - Align = DL.getABITypeAlignment(ObjectType); - } - } - } - } else if (const Argument *A = dyn_cast(this)) { - Align = A->getParamAlignment(); - - if (!Align && A->hasStructRetAttr()) { - // An sret parameter has at least the ABI alignment of the return type. - Type *EltTy = cast(A->getType())->getElementType(); - if (EltTy->isSized()) - Align = DL.getABITypeAlignment(EltTy); - } - } else if (const AllocaInst *AI = dyn_cast(this)) { - Align = AI->getAlignment(); - if (Align == 0) { - Type *AllocatedType = AI->getAllocatedType(); - if (AllocatedType->isSized()) - Align = DL.getPrefTypeAlignment(AllocatedType); - } - } else if (const auto *Call = dyn_cast(this)) { - Align = Call->getRetAlignment(); - if (Align == 0 && Call->getCalledFunction()) - Align = Call->getCalledFunction()->getAttributes().getRetAlignment(); - } else if (const LoadInst *LI = dyn_cast(this)) - if (MDNode *MD = LI->getMetadata(LLVMContext::MD_align)) { - ConstantInt *CI = mdconst::extract(MD->getOperand(0)); - Align = CI->getLimitedValue(); - } - - return Align; -} - const Value *Value::DoPHITranslation(const BasicBlock *CurBB, const BasicBlock *PredBB) const { auto *PN = dyn_cast(this); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h @@ -17,6 +17,7 @@ #include "AArch64RegisterInfo.h" #include "AArch64StackOffset.h" #include "llvm/ADT/Optional.h" +#include "llvm/Analysis/Loads.h" #include "llvm/CodeGen/MachineCombinerPattern.h" #include "llvm/CodeGen/TargetInstrInfo.h" diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -2188,7 +2188,7 @@ def alignedglobal : PatLeaf<(iPTR iPTR:$label), [{ if (auto *G = dyn_cast(N)) { const DataLayout &DL = MF->getDataLayout(); - unsigned Align = G->getGlobal()->getPointerAlignment(DL); + unsigned Align = getPointerAlignment(G->getGlobal(), DL); return Align >= 4 && G->getOffset() % 4 == 0; } if (auto *C = dyn_cast(N)) diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp --- a/llvm/lib/Transforms/IPO/Attributor.cpp +++ b/llvm/lib/Transforms/IPO/Attributor.cpp @@ -2058,8 +2058,9 @@ if (!Stripped && this == &AA) { // Use IR information if we did not strip anything. // TODO: track globally. - bool CanBeNull; - DerefBytes = Base->getPointerDereferenceableBytes(DL, CanBeNull); + bool CanBeNull, IsKnownDeref; + DerefBytes = + getPointerDereferenceableBytes(Base, DL, CanBeNull, IsKnownDeref); T.GlobalState.indicatePessimisticFixpoint(); } else { const DerefState &DS = static_cast(AA.getState()); @@ -2154,6 +2155,19 @@ takeKnownMaximum(Attr.getValueAsInt()); } + /// See AbstractAttribute::manifest(...). + ChangeStatus manifest(Attributor &A) override { + const Value &AssociatedVal = getAssociatedValue(); + Type *Ty = isa(AssociatedVal) + ? cast(AssociatedVal).getReturnType() + : AssociatedVal.getType()->getPointerElementType(); + if (Ty->isSized() && + getAssumed() <= A.getDataLayout().getABITypeAlignment(Ty)) + return ChangeStatus::UNCHANGED; + + return AAAlign::manifest(A); + } + /// See AbstractAttribute::getDeducedAttributes virtual void getDeducedAttributes(LLVMContext &Ctx, @@ -2208,7 +2222,7 @@ const auto &AA = A.getAAFor(*this, IRPosition::value(V)); if (!Stripped && this == &AA) { // Use only IR information if we did not strip anything. - T.takeKnownMaximum(V.getPointerAlignment(DL)); + T.takeKnownMaximum(getPointerAlignment(&V, DL)); T.indicatePessimisticFixpoint(); } else { // Use abstract attribute information. diff --git a/llvm/test/CodeGen/PowerPC/noPermuteFormasking.ll b/llvm/test/CodeGen/PowerPC/noPermuteFormasking.ll --- a/llvm/test/CodeGen/PowerPC/noPermuteFormasking.ll +++ b/llvm/test/CodeGen/PowerPC/noPermuteFormasking.ll @@ -1,6 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \ -; RUN: -verify-machineinstrs -O2 < %s | FileCheck %s +; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown -verify-machineinstrs -O2 < %s | FileCheck %s $test = comdat any ; Function Attrs: noinline nounwind @@ -8,10 +7,8 @@ ; CHECK-LABEL: test: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: ld 3, 0(3) -; CHECK-NEXT: cmpdi 1, 3, 0 -; CHECK-NEXT: andi. 4, 3, 3 -; CHECK-NEXT: crand 20, 2, 5 -; CHECK-NEXT: isel 3, 0, 3, 20 +; CHECK-NEXT: cmpdi 3, 0 +; CHECK-NEXT: isel 3, 0, 3, 1 ; CHECK-NEXT: addi 3, 3, -1 ; CHECK-NEXT: cmpldi 3, 3 ; CHECK-NEXT: bltlr+ 0 @@ -94,6 +91,13 @@ } define i64 @andis_no_cmp(i64 %a, i64 %b) { +; CHECK-LABEL: andis_no_cmp: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: andis. 5, 3, 1 +; CHECK-NEXT: li 5, 1 +; CHECK-NEXT: isel 4, 4, 5, 2 +; CHECK-NEXT: mulld 3, 4, 3 +; CHECK-NEXT: blr entry: %and = and i64 %a, 65536 %tobool = icmp eq i64 %and, 0 diff --git a/llvm/test/Transforms/FunctionAttrs/align.ll b/llvm/test/Transforms/FunctionAttrs/align.ll --- a/llvm/test/Transforms/FunctionAttrs/align.ll +++ b/llvm/test/Transforms/FunctionAttrs/align.ll @@ -7,8 +7,8 @@ ; TEST 1 -; ATTRIBUTOR: define align 8 i32* @test1(i32* returned align 8 %0) -define i32* @test1(i32* align 8 %0) #0 { +; ATTRIBUTOR: define align 16 i32* @test1(i32* returned align 16 %0) +define i32* @test1(i32* align 16 %0) #0 { ret i32* %0 } @@ -19,8 +19,8 @@ } ; TEST 3 -; ATTRIBUTOR: define align 4 i32* @test3(i32* align 8 %0, i32* align 4 %1, i1 %2) -define i32* @test3(i32* align 8 %0, i32* align 4 %1, i1 %2) #0 { +; ATTRIBUTOR: define align 16 i32* @test3(i32* align 32 %0, i32* align 16 %1, i1 %2) +define i32* @test3(i32* align 32 %0, i32* align 16 %1, i1 %2) #0 { %ret = select i1 %2, i32* %0, i32* %1 ret i32* %ret } @@ -34,18 +34,18 @@ ; TEST 5 declare i32* @unknown() -declare align 8 i32* @align8() +declare align 32 i32* @align32() -; ATTRIBUTOR: define align 8 i32* @test5_1() +; ATTRIBUTOR: define align 32 i32* @test5_1() define i32* @test5_1() { - %ret = tail call align 8 i32* @unknown() + %ret = tail call align 32 i32* @unknown() ret i32* %ret } -; ATTRIBUTOR: define align 8 i32* @test5_2() +; ATTRIBUTOR: define align 32 i32* @test5_2() define i32* @test5_2() { - %ret = tail call i32* @align8() + %ret = tail call i32* @align32() ret i32* %ret } @@ -78,17 +78,17 @@ ; return a?&a1: f1(&a2); ; } -@a1 = common global i8 0, align 8 -@a2 = common global i8 0, align 16 +@a1 = common global i8 0, align 16 +@a2 = common global i8 0, align 32 ; Function Attrs: nounwind readnone ssp uwtable define internal i8* @f1(i8* readnone %0) local_unnamed_addr #0 { -; ATTRIBUTOR: define internal nonnull align 8 dereferenceable(1) i8* @f1(i8* nonnull readnone align 8 dereferenceable(1) %0) +; ATTRIBUTOR: define internal nonnull align 16 dereferenceable(1) i8* @f1(i8* nonnull readnone align 16 dereferenceable(1) %0) %2 = icmp eq i8* %0, null br i1 %2, label %3, label %5 ;