diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h --- a/clang/lib/CodeGen/Address.h +++ b/clang/lib/CodeGen/Address.h @@ -22,52 +22,69 @@ namespace clang { namespace CodeGen { +// Indicates whether a pointer is known not to be null. +enum KnownNonNull_t { NotKnownNonNull, KnownNonNull }; + // We try to save some space by using 6 bits over two PointerIntPairs to store // the alignment. However, some arches don't support 3 bits in a PointerIntPair // so we fallback to storing the alignment separately. template = 8> class AddressImpl {}; template class AddressImpl { - llvm::Value *Pointer; + llvm::PointerIntPair PointerAndKnownNonNull; llvm::Type *ElementType; CharUnits Alignment; public: AddressImpl(llvm::Value *Pointer, llvm::Type *ElementType, - CharUnits Alignment) - : Pointer(Pointer), ElementType(ElementType), Alignment(Alignment) {} - llvm::Value *getPointer() const { return Pointer; } + CharUnits Alignment, KnownNonNull_t IsKnownNonNull) + : PointerAndKnownNonNull(Pointer, IsKnownNonNull), + ElementType(ElementType), Alignment(Alignment) {} + llvm::Value *getPointer() const { + return PointerAndKnownNonNull.getPointer(); + } llvm::Type *getElementType() const { return ElementType; } CharUnits getAlignment() const { return Alignment; } + KnownNonNull_t isKnownNonNull() const { + return (KnownNonNull_t)PointerAndKnownNonNull.getInt(); + } + void setKnownNonNull() { PointerAndKnownNonNull.setInt(true); } }; template class AddressImpl { - // Int portion stores upper 3 bits of the log of the alignment. + // Int portion stores the non-null bit and the upper 2 bits of the log of the + // alignment. llvm::PointerIntPair Pointer; // Int portion stores lower 3 bits of the log of the alignment. llvm::PointerIntPair ElementType; public: AddressImpl(llvm::Value *Pointer, llvm::Type *ElementType, - CharUnits Alignment) + CharUnits Alignment, KnownNonNull_t IsKnownNonNull) : Pointer(Pointer), ElementType(ElementType) { - if (Alignment.isZero()) + if (Alignment.isZero()) { + this->Pointer.setInt(IsKnownNonNull << 2); return; - // Currently the max supported alignment is much less than 1 << 63 and is + } + // Currently the max supported alignment is exactly 1 << 32 and is // guaranteed to be a power of 2, so we can store the log of the alignment - // into 6 bits. + // into 5 bits. assert(Alignment.isPowerOfTwo() && "Alignment cannot be zero"); auto AlignLog = llvm::Log2_64(Alignment.getQuantity()); - assert(AlignLog < (1 << 6) && "cannot fit alignment into 6 bits"); - this->Pointer.setInt(AlignLog >> 3); + assert(AlignLog < (1 << 5) && "cannot fit alignment into 5 bits"); + this->Pointer.setInt(IsKnownNonNull << 2 | AlignLog >> 3); this->ElementType.setInt(AlignLog & 7); } llvm::Value *getPointer() const { return Pointer.getPointer(); } llvm::Type *getElementType() const { return ElementType.getPointer(); } CharUnits getAlignment() const { - unsigned AlignLog = (Pointer.getInt() << 3) | ElementType.getInt(); + unsigned AlignLog = ((Pointer.getInt() & 0x3) << 3) | ElementType.getInt(); return CharUnits::fromQuantity(CharUnits::QuantityType(1) << AlignLog); } + KnownNonNull_t isKnownNonNull() const { + return (KnownNonNull_t)(!!(Pointer.getInt() & 0x4)); + } + void setKnownNonNull() { Pointer.setInt(Pointer.getInt() | 0x4); } }; /// An aligned address. @@ -75,11 +92,13 @@ AddressImpl A; protected: - Address(std::nullptr_t) : A(nullptr, nullptr, CharUnits::Zero()) {} + Address(std::nullptr_t) + : A(nullptr, nullptr, CharUnits::Zero(), NotKnownNonNull) {} public: - Address(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment) - : A(Pointer, ElementType, Alignment) { + Address(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull) + : A(Pointer, ElementType, Alignment, IsKnownNonNull) { assert(Pointer != nullptr && "Pointer cannot be null"); assert(ElementType != nullptr && "Element type cannot be null"); assert(llvm::cast(Pointer->getType()) @@ -124,14 +143,30 @@ /// Return address with different pointer, but same element type and /// alignment. - Address withPointer(llvm::Value *NewPointer) const { - return Address(NewPointer, getElementType(), getAlignment()); + Address withPointer(llvm::Value *NewPointer, + KnownNonNull_t IsKnownNonNull) const { + return Address(NewPointer, getElementType(), getAlignment(), + IsKnownNonNull); } /// Return address with different alignment, but same pointer and element /// type. Address withAlignment(CharUnits NewAlignment) const { - return Address(getPointer(), getElementType(), NewAlignment); + return Address(getPointer(), getElementType(), NewAlignment, + isKnownNonNull()); + } + + /// Whether the pointer is known not to be null. + KnownNonNull_t isKnownNonNull() const { + assert(isValid()); + return A.isKnownNonNull(); + } + + /// Set the non-null bit. + Address setKnownNonNull() { + assert(isValid()); + A.setKnownNonNull(); + return *this; } }; diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h --- a/clang/lib/CodeGen/CGBuilder.h +++ b/clang/lib/CodeGen/CGBuilder.h @@ -160,7 +160,8 @@ assert(cast(Ty)->isOpaqueOrPointeeTypeMatches( Addr.getElementType()) && "Should not change the element type"); - return Addr.withPointer(CreateAddrSpaceCast(Addr.getPointer(), Ty, Name)); + return Addr.withPointer(CreateAddrSpaceCast(Addr.getPointer(), Ty, Name), + Addr.isKnownNonNull()); } /// Cast the element type of the given address to a different type, @@ -169,7 +170,7 @@ const llvm::Twine &Name = "") { auto *PtrTy = Ty->getPointerTo(Addr.getAddressSpace()); return Address(CreateBitCast(Addr.getPointer(), PtrTy, Name), Ty, - Addr.getAlignment()); + Addr.getAlignment(), Addr.isKnownNonNull()); } using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast; @@ -178,7 +179,7 @@ const llvm::Twine &Name = "") { llvm::Value *Ptr = CreatePointerBitCastOrAddrSpaceCast(Addr.getPointer(), Ty, Name); - return Address(Ptr, ElementTy, Addr.getAlignment()); + return Address(Ptr, ElementTy, Addr.getAlignment(), Addr.isKnownNonNull()); } /// Given @@ -199,7 +200,7 @@ return Address( CreateStructGEP(Addr.getElementType(), Addr.getPointer(), Index, Name), ElTy->getElementType(Index), - Addr.getAlignment().alignmentAtOffset(Offset)); + Addr.getAlignment().alignmentAtOffset(Offset), Addr.isKnownNonNull()); } /// Given @@ -221,7 +222,8 @@ CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), {getSize(CharUnits::Zero()), getSize(Index)}, Name), ElTy->getElementType(), - Addr.getAlignment().alignmentAtOffset(Index * EltSize)); + Addr.getAlignment().alignmentAtOffset(Index * EltSize), + Addr.isKnownNonNull()); } /// Given @@ -237,8 +239,8 @@ return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), getSize(Index), Name), - ElTy, - Addr.getAlignment().alignmentAtOffset(Index * EltSize)); + ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize), + Addr.isKnownNonNull()); } /// Given @@ -255,7 +257,8 @@ return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), getSize(Index), Name), Addr.getElementType(), - Addr.getAlignment().alignmentAtOffset(Index * EltSize)); + Addr.getAlignment().alignmentAtOffset(Index * EltSize), + NotKnownNonNull); } /// Create GEP with single dynamic index. The address alignment is reduced @@ -270,7 +273,7 @@ return Address( CreateGEP(Addr.getElementType(), Addr.getPointer(), Index, Name), Addr.getElementType(), - Addr.getAlignment().alignmentOfArrayElement(EltSize)); + Addr.getAlignment().alignmentOfArrayElement(EltSize), NotKnownNonNull); } /// Given a pointer to i8, adjust it by a given constant offset. @@ -280,7 +283,8 @@ return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), getSize(Offset), Name), Addr.getElementType(), - Addr.getAlignment().alignmentAtOffset(Offset)); + Addr.getAlignment().alignmentAtOffset(Offset), + Addr.isKnownNonNull()); } Address CreateConstByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name = "") { @@ -288,7 +292,8 @@ return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), getSize(Offset), Name), Addr.getElementType(), - Addr.getAlignment().alignmentAtOffset(Offset)); + Addr.getAlignment().alignmentAtOffset(Offset), + NotKnownNonNull); } using CGBuilderBaseTy::CreateConstInBoundsGEP2_32; @@ -305,7 +310,8 @@ llvm_unreachable("offset of GEP with constants is always computable"); return Address(GEP, GEP->getResultElementType(), Addr.getAlignment().alignmentAtOffset( - CharUnits::fromQuantity(Offset.getSExtValue()))); + CharUnits::fromQuantity(Offset.getSExtValue())), + Addr.isKnownNonNull()); } using CGBuilderBaseTy::CreateMemCpy; @@ -369,7 +375,8 @@ using CGBuilderBaseTy::CreateLaunderInvariantGroup; Address CreateLaunderInvariantGroup(Address Addr) { - return Addr.withPointer(CreateLaunderInvariantGroup(Addr.getPointer())); + return Addr.withPointer(CreateLaunderInvariantGroup(Addr.getPointer()), + Addr.isKnownNonNull()); } }; diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -2804,7 +2804,7 @@ case ABIArgInfo::IndirectAliased: { assert(NumIRArgs == 1); Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty), - ArgI.getIndirectAlign()); + ArgI.getIndirectAlign(), KnownNonNull); if (!hasScalarEvaluationKind(Ty)) { // Aggregates and complex variables are accessed by reference. All we diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp --- a/clang/lib/CodeGen/CGClass.cpp +++ b/clang/lib/CodeGen/CGClass.cpp @@ -139,7 +139,7 @@ } llvm::Type *Ty = ConvertType(MD->getThisType()->getPointeeType()); - return Address(LoadCXXThis(), Ty, CXXThisAlignment); + return Address(LoadCXXThis(), Ty, CXXThisAlignment, KnownNonNull); } /// Emit the address of a field using a member data pointer. @@ -392,7 +392,7 @@ llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result"); PHI->addIncoming(Value.getPointer(), notNullBB); PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB); - Value = Value.withPointer(PHI); + Value = Value.withPointer(PHI, NotKnownNonNull); } return Value; diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp --- a/clang/lib/CodeGen/CGDecl.cpp +++ b/clang/lib/CodeGen/CGDecl.cpp @@ -2510,8 +2510,10 @@ CGM.getDataLayout().getAllocaAddrSpace()); auto DestAS = getContext().getTargetAddressSpace(DestLangAS); auto *T = DeclPtr.getElementType()->getPointerTo(DestAS); - DeclPtr = DeclPtr.withPointer(getTargetHooks().performAddrSpaceCast( - *this, V, SrcLangAS, DestLangAS, T, true)); + DeclPtr = + DeclPtr.withPointer(getTargetHooks().performAddrSpaceCast( + *this, V, SrcLangAS, DestLangAS, T, true), + DeclPtr.isKnownNonNull()); } // Push a destructor cleanup for this parameter if the ABI requires it. diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp --- a/clang/lib/CodeGen/CGException.cpp +++ b/clang/lib/CodeGen/CGException.cpp @@ -1842,7 +1842,7 @@ llvm::Value *ChildVar = Builder.CreateBitCast(RecoverCall, ParentVar.getType()); ChildVar->setName(ParentVar.getName()); - return ParentVar.withPointer(ChildVar); + return ParentVar.withPointer(ChildVar, KnownNonNull); } void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF, diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -72,7 +72,7 @@ llvm::Value *ArraySize) { auto Alloca = CreateTempAlloca(Ty, Name, ArraySize); Alloca->setAlignment(Align.getAsAlign()); - return Address(Alloca, Ty, Align); + return Address(Alloca, Ty, Align, KnownNonNull); } /// CreateTempAlloca - This creates a alloca and inserts it into the entry @@ -102,7 +102,7 @@ Ty->getPointerTo(DestAddrSpace), /*non-null*/ true); } - return Address(V, Ty, Align); + return Address(V, Ty, Align, KnownNonNull); } /// CreateTempAlloca - This creates an alloca and inserts it into the entry @@ -151,7 +151,7 @@ Result = Address( Builder.CreateBitCast(Result.getPointer(), VectorTy->getPointerTo()), - VectorTy, Result.getAlignment()); + VectorTy, Result.getAlignment(), KnownNonNull); } return Result; } @@ -1035,11 +1035,10 @@ // LValue Expression Emission //===----------------------------------------------------------------------===// -/// EmitPointerWithAlignment - Given an expression of pointer type, try to -/// derive a more accurate bound on the alignment of the pointer. -Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E, - LValueBaseInfo *BaseInfo, - TBAAAccessInfo *TBAAInfo) { +static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, + TBAAAccessInfo *TBAAInfo, + KnownNonNull_t IsKnownNonNull, + CodeGenFunction &CGF) { // We allow this with ObjC object pointers because of fragile ABIs. assert(E->getType()->isPointerType() || E->getType()->isObjCObjectPointerType()); @@ -1048,7 +1047,7 @@ // Casts: if (const CastExpr *CE = dyn_cast(E)) { if (const auto *ECE = dyn_cast(CE)) - CGM.EmitExplicitCastExprType(ECE, this); + CGF.CGM.EmitExplicitCastExprType(ECE, &CGF); switch (CE->getCastKind()) { // Non-converting casts (but not C's implicit conversion from void*). @@ -1061,49 +1060,51 @@ LValueBaseInfo InnerBaseInfo; TBAAAccessInfo InnerTBAAInfo; - Address Addr = EmitPointerWithAlignment(CE->getSubExpr(), - &InnerBaseInfo, - &InnerTBAAInfo); + Address Addr = CGF.EmitPointerWithAlignment( + CE->getSubExpr(), &InnerBaseInfo, &InnerTBAAInfo, IsKnownNonNull); if (BaseInfo) *BaseInfo = InnerBaseInfo; if (TBAAInfo) *TBAAInfo = InnerTBAAInfo; if (isa(CE)) { LValueBaseInfo TargetTypeBaseInfo; TBAAAccessInfo TargetTypeTBAAInfo; - CharUnits Align = CGM.getNaturalPointeeTypeAlignment( + CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment( E->getType(), &TargetTypeBaseInfo, &TargetTypeTBAAInfo); if (TBAAInfo) - *TBAAInfo = CGM.mergeTBAAInfoForCast(*TBAAInfo, - TargetTypeTBAAInfo); + *TBAAInfo = + CGF.CGM.mergeTBAAInfoForCast(*TBAAInfo, TargetTypeTBAAInfo); // If the source l-value is opaque, honor the alignment of the // casted-to type. if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) { if (BaseInfo) BaseInfo->mergeForCast(TargetTypeBaseInfo); - Addr = Address(Addr.getPointer(), Addr.getElementType(), Align); + Addr = Address(Addr.getPointer(), Addr.getElementType(), Align, + IsKnownNonNull); } } - if (SanOpts.has(SanitizerKind::CFIUnrelatedCast) && + if (CGF.SanOpts.has(SanitizerKind::CFIUnrelatedCast) && CE->getCastKind() == CK_BitCast) { if (auto PT = E->getType()->getAs()) - EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr, - /*MayBeNull=*/true, - CodeGenFunction::CFITCK_UnrelatedCast, - CE->getBeginLoc()); + CGF.EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr, + /*MayBeNull=*/true, + CodeGenFunction::CFITCK_UnrelatedCast, + CE->getBeginLoc()); } - llvm::Type *ElemTy = ConvertTypeForMem(E->getType()->getPointeeType()); - Addr = Builder.CreateElementBitCast(Addr, ElemTy); + llvm::Type *ElemTy = + CGF.ConvertTypeForMem(E->getType()->getPointeeType()); + Addr = CGF.Builder.CreateElementBitCast(Addr, ElemTy); if (CE->getCastKind() == CK_AddressSpaceConversion) - Addr = Builder.CreateAddrSpaceCast(Addr, ConvertType(E->getType())); + Addr = CGF.Builder.CreateAddrSpaceCast(Addr, + CGF.ConvertType(E->getType())); return Addr; } break; // Array-to-pointer decay. case CK_ArrayToPointerDecay: - return EmitArrayToPointerDecay(CE->getSubExpr(), BaseInfo, TBAAInfo); + return CGF.EmitArrayToPointerDecay(CE->getSubExpr(), BaseInfo, TBAAInfo); // Derived-to-base conversions. case CK_UncheckedDerivedToBase: @@ -1112,13 +1113,15 @@ // conservatively pretend that the complete object is of the base class // type. if (TBAAInfo) - *TBAAInfo = CGM.getTBAAAccessInfo(E->getType()); - Address Addr = EmitPointerWithAlignment(CE->getSubExpr(), BaseInfo); + *TBAAInfo = CGF.CGM.getTBAAAccessInfo(E->getType()); + Address Addr = CGF.EmitPointerWithAlignment( + CE->getSubExpr(), BaseInfo, nullptr, + (KnownNonNull_t)(IsKnownNonNull || + CE->getCastKind() == CK_UncheckedDerivedToBase)); auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl(); - return GetAddressOfBaseClass(Addr, Derived, - CE->path_begin(), CE->path_end(), - ShouldNullCheckClassCastValue(CE), - CE->getExprLoc()); + return CGF.GetAddressOfBaseClass( + Addr, Derived, CE->path_begin(), CE->path_end(), + CGF.ShouldNullCheckClassCastValue(CE), CE->getExprLoc()); } // TODO: Is there any reason to treat base-to-derived conversions @@ -1131,10 +1134,10 @@ // Unary &. if (const UnaryOperator *UO = dyn_cast(E)) { if (UO->getOpcode() == UO_AddrOf) { - LValue LV = EmitLValue(UO->getSubExpr()); + LValue LV = CGF.EmitLValue(UO->getSubExpr(), IsKnownNonNull); if (BaseInfo) *BaseInfo = LV.getBaseInfo(); if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo(); - return LV.getAddress(*this); + return LV.getAddress(CGF); } } @@ -1146,10 +1149,10 @@ case Builtin::BIaddressof: case Builtin::BI__addressof: case Builtin::BI__builtin_addressof: { - LValue LV = EmitLValue(Call->getArg(0)); + LValue LV = CGF.EmitLValue(Call->getArg(0), IsKnownNonNull); if (BaseInfo) *BaseInfo = LV.getBaseInfo(); if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo(); - return LV.getAddress(*this); + return LV.getAddress(CGF); } } } @@ -1158,9 +1161,21 @@ // Otherwise, use the alignment of the type. CharUnits Align = - CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo); - llvm::Type *ElemTy = ConvertTypeForMem(E->getType()->getPointeeType()); - return Address(EmitScalarExpr(E), ElemTy, Align); + CGF.CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo); + llvm::Type *ElemTy = CGF.ConvertTypeForMem(E->getType()->getPointeeType()); + return Address(CGF.EmitScalarExpr(E), ElemTy, Align, IsKnownNonNull); +} + +/// EmitPointerWithAlignment - Given an expression of pointer type, try to +/// derive a more accurate bound on the alignment of the pointer. +Address CodeGenFunction::EmitPointerWithAlignment( + const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo, + KnownNonNull_t IsKnownNonNull) { + Address Addr = + ::EmitPointerWithAlignment(E, BaseInfo, TBAAInfo, IsKnownNonNull, *this); + if (IsKnownNonNull && !Addr.isKnownNonNull()) + Addr.setKnownNonNull(); + return Addr; } llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) { @@ -1270,7 +1285,16 @@ /// type of the same size of the lvalue's type. If the lvalue has a variable /// length type, this is not possible. /// -LValue CodeGenFunction::EmitLValue(const Expr *E) { +LValue CodeGenFunction::EmitLValue(const Expr *E, + KnownNonNull_t IsKnownNonNull) { + LValue LV = EmitLValueHelper(E, IsKnownNonNull); + if (IsKnownNonNull && !LV.isKnownNonNull()) + LV.setKnownNonNull(); + return LV; +} + +LValue CodeGenFunction::EmitLValueHelper(const Expr *E, + KnownNonNull_t IsKnownNonNull) { ApplyDebugLocation DL(*this, E); switch (E->getStmtClass()) { default: return EmitUnsupportedLValue(E, "l-value expression"); @@ -1298,7 +1322,8 @@ case Expr::UserDefinedLiteralClass: return EmitCallExprLValue(cast(E)); case Expr::CXXRewrittenBinaryOperatorClass: - return EmitLValue(cast(E)->getSemanticForm()); + return EmitLValue(cast(E)->getSemanticForm(), + IsKnownNonNull); case Expr::VAArgExprClass: return EmitVAArgExprLValue(cast(E)); case Expr::DeclRefExprClass: @@ -1311,12 +1336,13 @@ ->getPointeeType(); return MakeNaturalAlignAddrLValue(Result, RetType); } - return EmitLValue(cast(E)->getSubExpr()); + return EmitLValue(cast(E)->getSubExpr(), IsKnownNonNull); } case Expr::ParenExprClass: - return EmitLValue(cast(E)->getSubExpr()); + return EmitLValue(cast(E)->getSubExpr(), IsKnownNonNull); case Expr::GenericSelectionExprClass: - return EmitLValue(cast(E)->getResultExpr()); + return EmitLValue(cast(E)->getResultExpr(), + IsKnownNonNull); case Expr::PredefinedExprClass: return EmitPredefinedLValue(cast(E)); case Expr::StringLiteralClass: @@ -1340,15 +1366,16 @@ case Expr::ExprWithCleanupsClass: { const auto *cleanups = cast(E); RunCleanupsScope Scope(*this); - LValue LV = EmitLValue(cleanups->getSubExpr()); + LValue LV = EmitLValue(cleanups->getSubExpr(), IsKnownNonNull); if (LV.isSimple()) { // Defend against branches out of gnu statement expressions surrounded by // cleanups. Address Addr = LV.getAddress(*this); llvm::Value *V = Addr.getPointer(); Scope.ForceCleanup({&V}); - return LValue::MakeAddr(Addr.withPointer(V), LV.getType(), getContext(), - LV.getBaseInfo(), LV.getTBAAInfo()); + return LValue::MakeAddr(Addr.withPointer(V, Addr.isKnownNonNull()), + LV.getType(), getContext(), LV.getBaseInfo(), + LV.getTBAAInfo()); } // FIXME: Is it possible to create an ExprWithCleanups that produces a // bitfield lvalue or some other non-simple lvalue? @@ -1358,12 +1385,12 @@ case Expr::CXXDefaultArgExprClass: { auto *DAE = cast(E); CXXDefaultArgExprScope Scope(*this, DAE); - return EmitLValue(DAE->getExpr()); + return EmitLValue(DAE->getExpr(), IsKnownNonNull); } case Expr::CXXDefaultInitExprClass: { auto *DIE = cast(E); CXXDefaultInitExprScope Scope(*this, DIE); - return EmitLValue(DIE->getExpr()); + return EmitLValue(DIE->getExpr(), IsKnownNonNull); } case Expr::CXXTypeidExprClass: return EmitCXXTypeidLValue(cast(E)); @@ -1395,11 +1422,12 @@ case Expr::BinaryConditionalOperatorClass: return EmitConditionalOperatorLValue(cast(E)); case Expr::ChooseExprClass: - return EmitLValue(cast(E)->getChosenSubExpr()); + return EmitLValue(cast(E)->getChosenSubExpr(), IsKnownNonNull); case Expr::OpaqueValueExprClass: return EmitOpaqueValueLValue(cast(E)); case Expr::SubstNonTypeTemplateParmExprClass: - return EmitLValue(cast(E)->getReplacement()); + return EmitLValue(cast(E)->getReplacement(), + IsKnownNonNull); case Expr::ImplicitCastExprClass: case Expr::CStyleCastExprClass: case Expr::CXXFunctionalCastExprClass: @@ -1691,7 +1719,8 @@ bool isNontemporal) { if (auto *GV = dyn_cast(Addr.getPointer())) if (GV->isThreadLocal()) - Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV)); + Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV), + NotKnownNonNull); if (const auto *ClangVecTy = Ty->getAs()) { // Boolean vectors use `iN` as storage type. @@ -1839,7 +1868,8 @@ bool isInit, bool isNontemporal) { if (auto *GV = dyn_cast(Addr.getPointer())) if (GV->isThreadLocal()) - Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV)); + Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV), + NotKnownNonNull); llvm::Type *SrcTy = Value->getType(); if (const auto *ClangVecTy = Ty->getAs()) { @@ -2848,8 +2878,8 @@ // Handle threadlocal function locals. if (VD->getTLSKind() != VarDecl::TLS_None) - addr = - addr.withPointer(Builder.CreateThreadLocalAddress(addr.getPointer())); + addr = addr.withPointer( + Builder.CreateThreadLocalAddress(addr.getPointer()), NotKnownNonNull); // Check for OpenMP threadprivate variables. if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd && @@ -5106,7 +5136,7 @@ functionType = ptrType->getPointeeType(); } else { functionType = E->getType(); - calleePtr = EmitLValue(E).getPointer(*this); + calleePtr = EmitLValue(E, KnownNonNull).getPointer(*this); } assert(functionType->isFunctionType()); diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp --- a/clang/lib/CodeGen/CGExprCXX.cpp +++ b/clang/lib/CodeGen/CGExprCXX.cpp @@ -443,9 +443,9 @@ // Emit the 'this' pointer. Address This = Address::invalid(); if (BO->getOpcode() == BO_PtrMemI) - This = EmitPointerWithAlignment(BaseExpr); + This = EmitPointerWithAlignment(BaseExpr, nullptr, nullptr, KnownNonNull); else - This = EmitLValue(BaseExpr).getAddress(*this); + This = EmitLValue(BaseExpr, KnownNonNull).getAddress(*this); EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.getPointer(), QualType(MPT->getClass(), 0)); @@ -2071,6 +2071,7 @@ Builder.CreateCondBr(IsNull, DeleteEnd, DeleteNotNull); EmitBlock(DeleteNotNull); + Ptr.setKnownNonNull(); QualType DeleteTy = E->getDestroyedType(); @@ -2103,7 +2104,8 @@ Ptr = Address(Builder.CreateInBoundsGEP(Ptr.getElementType(), Ptr.getPointer(), GEP, "del.first"), - ConvertTypeForMem(DeleteTy), Ptr.getAlignment()); + ConvertTypeForMem(DeleteTy), Ptr.getAlignment(), + Ptr.isKnownNonNull()); } assert(ConvertTypeForMem(DeleteTy) == Ptr.getElementType()); diff --git a/clang/lib/CodeGen/CGNonTrivialStruct.cpp b/clang/lib/CodeGen/CGNonTrivialStruct.cpp --- a/clang/lib/CodeGen/CGNonTrivialStruct.cpp +++ b/clang/lib/CodeGen/CGNonTrivialStruct.cpp @@ -327,7 +327,7 @@ CodeGenFunction *CGF) { return std::array{ {Address(CGF->Builder.CreateLoad(CGF->GetAddrOfLocalVar(Args[Ints])), - CGF->VoidPtrTy, Alignments[Ints])...}}; + CGF->VoidPtrTy, Alignments[Ints], KnownNonNull)...}}; } // Template classes that are used as bases for classes that emit special diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -963,7 +963,7 @@ Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( Addr, OriginalBaseAddress.getType()); - return OriginalBaseAddress.withPointer(Addr); + return OriginalBaseAddress.withPointer(Addr, NotKnownNonNull); } static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) { @@ -4670,7 +4670,7 @@ llvm::PHINode *ElementPHI = CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast"); ElementPHI->addIncoming(Begin.getPointer(), EntryBB); - Begin = Begin.withPointer(ElementPHI); + Begin = Begin.withPointer(ElementPHI, KnownNonNull); Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); // deps[i].flags = NewDepKind; diff --git a/clang/lib/CodeGen/CGValue.h b/clang/lib/CodeGen/CGValue.h --- a/clang/lib/CodeGen/CGValue.h +++ b/clang/lib/CodeGen/CGValue.h @@ -225,6 +225,8 @@ // this lvalue. bool Nontemporal : 1; + bool IsKnownNonNull : 1l; + LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; @@ -333,24 +335,35 @@ LValueBaseInfo getBaseInfo() const { return BaseInfo; } void setBaseInfo(LValueBaseInfo Info) { BaseInfo = Info; } + KnownNonNull_t isKnownNonNull() const { + return (KnownNonNull_t)IsKnownNonNull; + } + LValue setKnownNonNull() { + IsKnownNonNull = true; + return *this; + } + // simple lvalue llvm::Value *getPointer(CodeGenFunction &CGF) const { assert(isSimple()); return V; } Address getAddress(CodeGenFunction &CGF) const { - return Address(getPointer(CGF), ElementType, getAlignment()); + return Address(getPointer(CGF), ElementType, getAlignment(), + (KnownNonNull_t)isKnownNonNull()); } void setAddress(Address address) { assert(isSimple()); V = address.getPointer(); ElementType = address.getElementType(); Alignment = address.getAlignment().getQuantity(); + IsKnownNonNull = address.isKnownNonNull(); } // vector elt lvalue Address getVectorAddress() const { - return Address(getVectorPointer(), ElementType, getAlignment()); + return Address(getVectorPointer(), ElementType, getAlignment(), + (KnownNonNull_t)isKnownNonNull()); } llvm::Value *getVectorPointer() const { assert(isVectorElt()); @@ -362,7 +375,8 @@ } Address getMatrixAddress() const { - return Address(getMatrixPointer(), ElementType, getAlignment()); + return Address(getMatrixPointer(), ElementType, getAlignment(), + (KnownNonNull_t)isKnownNonNull()); } llvm::Value *getMatrixPointer() const { assert(isMatrixElt()); @@ -375,7 +389,8 @@ // extended vector elements. Address getExtVectorAddress() const { - return Address(getExtVectorPointer(), ElementType, getAlignment()); + return Address(getExtVectorPointer(), ElementType, getAlignment(), + (KnownNonNull_t)isKnownNonNull()); } llvm::Value *getExtVectorPointer() const { assert(isExtVectorElt()); @@ -388,7 +403,8 @@ // bitfield lvalue Address getBitFieldAddress() const { - return Address(getBitFieldPointer(), ElementType, getAlignment()); + return Address(getBitFieldPointer(), ElementType, getAlignment(), + (KnownNonNull_t)isKnownNonNull()); } llvm::Value *getBitFieldPointer() const { assert(isBitField()); return V; } const CGBitFieldInfo &getBitFieldInfo() const { @@ -409,6 +425,7 @@ assert(address.getPointer()->getType()->isPointerTy()); R.V = address.getPointer(); R.ElementType = address.getElementType(); + R.IsKnownNonNull = address.isKnownNonNull(); R.Initialize(type, qs, address.getAlignment(), BaseInfo, TBAAInfo); return R; } @@ -421,6 +438,7 @@ R.V = vecAddress.getPointer(); R.ElementType = vecAddress.getElementType(); R.VectorIdx = Idx; + R.IsKnownNonNull = vecAddress.isKnownNonNull(); R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(), BaseInfo, TBAAInfo); return R; @@ -434,6 +452,7 @@ R.V = vecAddress.getPointer(); R.ElementType = vecAddress.getElementType(); R.VectorElts = Elts; + R.IsKnownNonNull = vecAddress.isKnownNonNull(); R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(), BaseInfo, TBAAInfo); return R; @@ -453,6 +472,7 @@ R.V = Addr.getPointer(); R.ElementType = Addr.getElementType(); R.BitFieldInfo = &Info; + R.IsKnownNonNull = Addr.isKnownNonNull(); R.Initialize(type, type.getQualifiers(), Addr.getAlignment(), BaseInfo, TBAAInfo); return R; @@ -464,6 +484,7 @@ R.LVType = GlobalReg; R.V = V; R.ElementType = nullptr; + R.IsKnownNonNull = true; R.Initialize(type, type.getQualifiers(), alignment, LValueBaseInfo(AlignmentSource::Decl), TBAAAccessInfo()); return R; @@ -477,6 +498,7 @@ R.V = matAddress.getPointer(); R.ElementType = matAddress.getElementType(); R.VectorIdx = Idx; + R.IsKnownNonNull = matAddress.isKnownNonNull(); R.Initialize(type, type.getQualifiers(), matAddress.getAlignment(), BaseInfo, TBAAInfo); return R; @@ -579,6 +601,8 @@ Overlap_t mayOverlap, IsZeroed_t isZeroed = IsNotZeroed, IsSanitizerChecked_t isChecked = IsNotSanitizerChecked) { + if (addr.isValid()) + addr.setKnownNonNull(); return AggValueSlot(addr, quals, isDestructed, needsGC, isZeroed, isAliased, mayOverlap, isChecked); } diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -3164,7 +3164,8 @@ Address getIndirectAddress() const { assert(isIndirect()); - return Address(Value, ElementType, CharUnits::fromQuantity(Alignment)); + return Address(Value, ElementType, CharUnits::fromQuantity(Alignment), + KnownNonNull); } }; @@ -3771,8 +3772,13 @@ /// an LLVM type of the same size of the lvalue's type. If the lvalue has a /// variable length type, this is not possible. /// - LValue EmitLValue(const Expr *E); + LValue EmitLValue(const Expr *E, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull); +private: + LValue EmitLValueHelper(const Expr *E, KnownNonNull_t IsKnownNonNull); + +public: /// Same as EmitLValue but additionally we generate checking code to /// guard against undefined behavior. This is only suitable when we know /// that the address will be used to access the object. @@ -4783,9 +4789,10 @@ /// into the address of a local variable. In such a case, it's quite /// reasonable to just ignore the returned alignment when it isn't from an /// explicit source. - Address EmitPointerWithAlignment(const Expr *Addr, - LValueBaseInfo *BaseInfo = nullptr, - TBAAAccessInfo *TBAAInfo = nullptr); + Address + EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo = nullptr, + TBAAAccessInfo *TBAAInfo = nullptr, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull); /// If \p E references a parameter with pass_object_size info or a constant /// array size modifier, emit the object size divided by the size of \p EltTy. diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -1104,8 +1104,9 @@ auto AI = CurFn->arg_begin(); if (CurFnInfo->getReturnInfo().isSRetAfterThis()) ++AI; - ReturnValue = Address(&*AI, ConvertType(RetTy), - CurFnInfo->getReturnInfo().getIndirectAlign()); + ReturnValue = + Address(&*AI, ConvertType(RetTy), + CurFnInfo->getReturnInfo().getIndirectAlign(), KnownNonNull); if (!CurFnInfo->getReturnInfo().getIndirectByVal()) { ReturnValuePointer = CreateDefaultAlignTempAlloca(Int8PtrTy, "result.ptr"); @@ -1125,8 +1126,8 @@ cast(Addr)->getResultElementType(); ReturnValuePointer = Address(Addr, Ty, getPointerAlign()); Addr = Builder.CreateAlignedLoad(Ty, Addr, getPointerAlign(), "agg.result"); - ReturnValue = - Address(Addr, ConvertType(RetTy), CGM.getNaturalTypeAlignment(RetTy)); + ReturnValue = Address(Addr, ConvertType(RetTy), + CGM.getNaturalTypeAlignment(RetTy), KnownNonNull); } else { ReturnValue = CreateIRTemp(RetTy, "retval");