diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp --- a/llvm/lib/Analysis/ConstantFolding.cpp +++ b/llvm/lib/Analysis/ConstantFolding.cpp @@ -327,10 +327,25 @@ const DataLayout &DL) { do { Type *SrcTy = C->getType(); + uint64_t DestSize = DL.getTypeSizeInBits(DestTy); + uint64_t SrcSize = DL.getTypeSizeInBits(SrcTy); + if (SrcSize < DestSize) + return nullptr; + + // Catch the obvious splat cases (since all-zeros can coerce non-integral + // pointers legally). + if (C->isNullValue() && !DestTy->isX86_MMXTy()) + return Constant::getNullValue(DestTy); + if (C->isAllOnesValue() && !DestTy->isX86_MMXTy() && + !DestTy->isPtrOrPtrVectorTy()) // Don't get ones for ptr types! + return Constant::getAllOnesValue(DestTy); // If the type sizes are the same and a cast is legal, just directly // cast the constant. - if (DL.getTypeSizeInBits(DestTy) == DL.getTypeSizeInBits(SrcTy)) { + // But be careful not to coerce non-integral pointers illegally. + if (SrcSize == DestSize && + DL.isNonIntegralPointerType(SrcTy->getScalarType()) == + DL.isNonIntegralPointerType(DestTy->getScalarType())) { Instruction::CastOps Cast = Instruction::BitCast; // If we are going from a pointer to int or vice versa, we spell the cast // differently. diff --git a/llvm/lib/Transforms/Utils/VNCoercion.cpp b/llvm/lib/Transforms/Utils/VNCoercion.cpp --- a/llvm/lib/Transforms/Utils/VNCoercion.cpp +++ b/llvm/lib/Transforms/Utils/VNCoercion.cpp @@ -11,10 +11,8 @@ namespace llvm { namespace VNCoercion { -/// Return true if coerceAvailableValueToLoadType will succeed. -bool canCoerceMustAliasedValueToLoad(Value *StoredVal, Type *LoadTy, +bool canCoerceMustAliasedValueToLoad(Type *StoredTy, Type *LoadTy, const DataLayout &DL) { - Type *StoredTy = StoredVal->getType(); if (StoredTy == LoadTy) return true; @@ -35,19 +33,24 @@ return false; // Don't coerce non-integral pointers to integers or vice versa. - if (DL.isNonIntegralPointerType(StoredVal->getType()->getScalarType()) != - DL.isNonIntegralPointerType(LoadTy->getScalarType())) { - // As a special case, allow coercion of memset used to initialize - // an array w/null. Despite non-integral pointers not generally having a - // specific bit pattern, we do assume null is zero. - if (auto *CI = dyn_cast(StoredVal)) - return CI->isNullValue(); + if (DL.isNonIntegralPointerType(StoredTy->getScalarType()) != + DL.isNonIntegralPointerType(LoadTy->getScalarType())) return false; - } - + return true; } +/// Return true if coerceAvailableValueToLoadType will succeed. +bool canCoerceMustAliasedValueToLoad(Value *StoredVal, Type *LoadTy, + const DataLayout &DL) { + Type *StoredTy = StoredVal->getType(); + if (auto *CI = dyn_cast(StoredVal)) + if (CI->isNullValue()) + if (StoredTy == LoadTy || DL.getTypeSizeInBits(StoredTy) >= DL.getTypeSizeInBits(LoadTy)) + return true; + return canCoerceMustAliasedValueToLoad(StoredTy, LoadTy, DL); +} + template static T *coerceAvailableValueToLoadTypeHelper(T *StoredVal, Type *LoadedTy, HelperClass &Helper, @@ -164,11 +167,6 @@ Value *WritePtr, uint64_t WriteSizeInBits, const DataLayout &DL) { - // If the loaded or stored value is a first class array or struct, don't try - // to transform them. We need to be able to bitcast to integer. - if (LoadTy->isStructTy() || LoadTy->isArrayTy()) - return -1; - int64_t StoreOffset = 0, LoadOffset = 0; Value *StoreBase = GetPointerBaseWithConstantOffset(WritePtr, StoreOffset, DL); @@ -218,21 +216,9 @@ int analyzeLoadFromClobberingStore(Type *LoadTy, Value *LoadPtr, StoreInst *DepSI, const DataLayout &DL) { auto *StoredVal = DepSI->getValueOperand(); - - // Cannot handle reading from store of first-class aggregate yet. - if (StoredVal->getType()->isStructTy() || - StoredVal->getType()->isArrayTy()) + if (!canCoerceMustAliasedValueToLoad(StoredVal, LoadTy, DL)) return -1; - // Don't coerce non-integral pointers to integers or vice versa. - if (DL.isNonIntegralPointerType(StoredVal->getType()->getScalarType()) != - DL.isNonIntegralPointerType(LoadTy->getScalarType())) { - // Allow casts of zero values to null as a special case - auto *CI = dyn_cast(StoredVal); - if (!CI || !CI->isNullValue()) - return -1; - } - Value *StorePtr = DepSI->getPointerOperand(); uint64_t StoreSize = DL.getTypeSizeInBits(DepSI->getValueOperand()->getType()); @@ -245,23 +231,19 @@ /// the other load can feed into the second load. int analyzeLoadFromClobberingLoad(Type *LoadTy, Value *LoadPtr, LoadInst *DepLI, const DataLayout &DL) { - // Cannot handle reading from store of first-class aggregate yet. - if (DepLI->getType()->isStructTy() || DepLI->getType()->isArrayTy()) - return -1; - - // Don't coerce non-integral pointers to integers or vice versa. - if (DL.isNonIntegralPointerType(DepLI->getType()->getScalarType()) != - DL.isNonIntegralPointerType(LoadTy->getScalarType())) - return -1; - Value *DepPtr = DepLI->getPointerOperand(); - uint64_t DepSize = DL.getTypeSizeInBits(DepLI->getType()); - int R = analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, DepPtr, DepSize, DL); - if (R != -1) - return R; + if (canCoerceMustAliasedValueToLoad(DepLI->getType(), LoadTy, DL)) { + uint64_t DepSize = DL.getTypeSizeInBits(DepLI->getType()); + int R = analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, DepPtr, DepSize, DL); + if (R != -1) + return R; + } - // If we have a load/load clobber an DepLI can be widened to cover this load, + // If we have a load/load clobber and DepLI can be widened to cover this load, // then we should widen it! + if (!canCoerceMustAliasedValueToLoad(LoadTy, DepLI->getType(), DL)) + return -1; + int64_t LoadOffs = 0; const Value *LoadBase = GetPointerBaseWithConstantOffset(LoadPtr, LoadOffs, DL); @@ -291,13 +273,16 @@ // If this is memset, we just need to see if the offset is valid in the size // of the memset.. if (MI->getIntrinsicID() == Intrinsic::memset) { - if (DL.isNonIntegralPointerType(LoadTy->getScalarType())) { - auto *CI = dyn_cast(cast(MI)->getValue()); - if (!CI || !CI->isZero()) - return -1; - } - return analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, MI->getDest(), - MemSizeInBits, DL); + Value *StoredVal = cast(MI)->getValue(); + if (auto *CI = dyn_cast(StoredVal)) + if (CI->isNullValue()) + return analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, MI->getDest(), + MemSizeInBits, DL); + Type *StoreTy = IntegerType::get(LoadTy->getContext(), MemSizeInBits); + if (canCoerceMustAliasedValueToLoad(StoreTy, LoadTy, DL)) + return analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, MI->getDest(), + MemSizeInBits, DL); + return -1; } // If we have a memcpy/memmove, the only case we can handle is if this is a @@ -319,21 +304,17 @@ if (Offset == -1) return Offset; - // Don't coerce non-integral pointers to integers or vice versa, and the - // memtransfer is implicitly a raw byte code - if (DL.isNonIntegralPointerType(LoadTy->getScalarType())) - // TODO: Can allow nullptrs from constant zeros - return -1; - unsigned AS = Src->getType()->getPointerAddressSpace(); // Otherwise, see if we can constant fold a load from the constant with the // offset applied as appropriate. - Src = - ConstantExpr::getBitCast(Src, Type::getInt8PtrTy(Src->getContext(), AS)); - Constant *OffsetCst = - ConstantInt::get(Type::getInt64Ty(Src->getContext()), (unsigned)Offset); - Src = ConstantExpr::getGetElementPtr(Type::getInt8Ty(Src->getContext()), Src, - OffsetCst); + if (Offset) { + Src = + ConstantExpr::getBitCast(Src, Type::getInt8PtrTy(Src->getContext(), AS)); + Constant *OffsetCst = + ConstantInt::get(Type::getInt64Ty(Src->getContext()), (unsigned)Offset); + Src = ConstantExpr::getGetElementPtr(Type::getInt8Ty(Src->getContext()), Src, + OffsetCst); + } Src = ConstantExpr::getBitCast(Src, PointerType::get(LoadTy, AS)); if (ConstantFoldLoadFromConstPtr(Src, LoadTy, DL)) return Offset; @@ -345,6 +326,9 @@ HelperClass &Helper, const DataLayout &DL) { LLVMContext &Ctx = SrcVal->getType()->getContext(); + if (auto *CI = dyn_cast(GetUnderlyingObject(SrcVal, DL))) + if (CI->isNullValue()) + return Constant::getNullValue(LoadTy); // If two pointers are in the same address space, they have the same size, // so we don't need to do any truncation, etc. This avoids introducing @@ -472,6 +456,12 @@ // memset(P, 'x', 1234) -> splat('x'), even if x is a variable, and // independently of what the offset is. T *Val = cast(MSI->getValue()); + if (auto *CI = dyn_cast(Val)) { + // memset(P, '\0', 1234) -> just directly create the null value for *P + // by-passing any later validity checks + if (CI->isNullValue()) + return Constant::getNullValue(LoadTy); + } if (LoadSize != 1) Val = Helper.CreateZExtOrBitCast(Val, IntegerType::get(Ctx, LoadSize * 8)); @@ -500,16 +490,18 @@ // Otherwise, this is a memcpy/memmove from a constant global. MemTransferInst *MTI = cast(SrcInst); Constant *Src = cast(MTI->getSource()); - unsigned AS = Src->getType()->getPointerAddressSpace(); + unsigned AS = Src->getType()->getPointerAddressSpace(); // Otherwise, see if we can constant fold a load from the constant with the // offset applied as appropriate. - Src = - ConstantExpr::getBitCast(Src, Type::getInt8PtrTy(Src->getContext(), AS)); - Constant *OffsetCst = - ConstantInt::get(Type::getInt64Ty(Src->getContext()), (unsigned)Offset); - Src = ConstantExpr::getGetElementPtr(Type::getInt8Ty(Src->getContext()), Src, - OffsetCst); + if (Offset) { + Src = + ConstantExpr::getBitCast(Src, Type::getInt8PtrTy(Src->getContext(), AS)); + Constant *OffsetCst = + ConstantInt::get(Type::getInt64Ty(Src->getContext()), (unsigned)Offset); + Src = ConstantExpr::getGetElementPtr(Type::getInt8Ty(Src->getContext()), Src, + OffsetCst); + } Src = ConstantExpr::getBitCast(Src, PointerType::get(LoadTy, AS)); return ConstantFoldLoadFromConstPtr(Src, LoadTy, DL); } diff --git a/llvm/test/Transforms/GVN/non-integral-pointers.ll b/llvm/test/Transforms/GVN/non-integral-pointers.ll --- a/llvm/test/Transforms/GVN/non-integral-pointers.ll +++ b/llvm/test/Transforms/GVN/non-integral-pointers.ll @@ -169,7 +169,14 @@ ret i8 addrspace(4)* %ref } + + @NonZeroConstant = constant <4 x i64> +@NonZeroConstant2 = constant <4 x i64 addrspace(4)*> < + i64 addrspace(4)* getelementptr (i64, i64 addrspace(4)* null, i32 3), + i64 addrspace(4)* getelementptr (i64, i64 addrspace(4)* null, i32 3), + i64 addrspace(4)* getelementptr (i64, i64 addrspace(4)* null, i32 3), + i64 addrspace(4)* getelementptr (i64, i64 addrspace(4)* null, i32 3)> @ZeroConstant = constant <4 x i64> zeroinitializer @@ -190,6 +197,54 @@ ret i8 addrspace(4)* %ref } +define i64 addrspace(4)* @neg_forward_memcopy2(i64 addrspace(4)* addrspace(4)* %loc) { +; CHECK-LABEL: @neg_forward_memcopy2( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i64 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)* +; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64>* @NonZeroConstant to i8*), i64 8, i1 false) +; CHECK-NEXT: [[REF:%.*]] = load i64 addrspace(4)*, i64 addrspace(4)* addrspace(4)* [[LOC]] +; CHECK-NEXT: ret i64 addrspace(4)* [[REF]] +; +entry: + %loc.bc = bitcast i64 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)* + %src.bc = bitcast <4 x i64>* @NonZeroConstant to i8* + call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 8, i1 false) + %ref = load i64 addrspace(4)*, i64 addrspace(4)* addrspace(4)* %loc + ret i64 addrspace(4)* %ref +} + +; TODO: missed optimization +define i8 addrspace(4)* @forward_memcopy(i8 addrspace(4)* addrspace(4)* %loc) { +; CHECK-LABEL: @forward_memcopy( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)* +; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8*), i64 8, i1 false) +; CHECK-NEXT: [[REF:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc +; CHECK-NEXT: ret i8 addrspace(4)* [[REF]] +; +entry: + %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)* + %src.bc = bitcast <4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8* + call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 8, i1 false) + %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc + ret i8 addrspace(4)* %ref +} + +define i64 addrspace(4)* @forward_memcopy2(i64 addrspace(4)* addrspace(4)* %loc) { +; CHECK-LABEL: @forward_memcopy2( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i64 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)* +; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8*), i64 8, i1 false) +; CHECK-NEXT: ret i64 addrspace(4)* getelementptr (i64, i64 addrspace(4)* null, i32 3) +; +entry: + %loc.bc = bitcast i64 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)* + %src.bc = bitcast <4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8* + call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 8, i1 false) + %ref = load i64 addrspace(4)*, i64 addrspace(4)* addrspace(4)* %loc + ret i64 addrspace(4)* %ref +} + define <1 x i8 addrspace(4)*> @neg_forward_memcpy_vload(<1 x i8 addrspace(4)*> addrspace(4)* %loc) { ; CHECK-LABEL: @neg_forward_memcpy_vload( ; CHECK-NEXT: entry: @@ -206,16 +261,62 @@ ret <1 x i8 addrspace(4)*> %ref } +define <4 x i64 addrspace(4)*> @neg_forward_memcpy_vload2(<4 x i64 addrspace(4)*> addrspace(4)* %loc) { +; CHECK-LABEL: @neg_forward_memcpy_vload2( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast <4 x i64 addrspace(4)*> addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)* +; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64>* @NonZeroConstant to i8*), i64 32, i1 false) +; CHECK-NEXT: [[REF:%.*]] = load <4 x i64 addrspace(4)*>, <4 x i64 addrspace(4)*> addrspace(4)* [[LOC]] +; CHECK-NEXT: ret <4 x i64 addrspace(4)*> [[REF]] +; +entry: + %loc.bc = bitcast <4 x i64 addrspace(4)*> addrspace(4)* %loc to i8 addrspace(4)* + %src.bc = bitcast <4 x i64>* @NonZeroConstant to i8* + call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 32, i1 false) + %ref = load <4 x i64 addrspace(4)*>, <4 x i64 addrspace(4)*> addrspace(4)* %loc + ret <4 x i64 addrspace(4)*> %ref +} + +define <4 x i64> @neg_forward_memcpy_vload3(<4 x i64> addrspace(4)* %loc) { +; CHECK-LABEL: @neg_forward_memcpy_vload3( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast <4 x i64> addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)* +; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8*), i64 32, i1 false) +; CHECK-NEXT: [[REF:%.*]] = load <4 x i64>, <4 x i64> addrspace(4)* [[LOC]] +; CHECK-NEXT: ret <4 x i64> [[REF]] +; +entry: + %loc.bc = bitcast <4 x i64> addrspace(4)* %loc to i8 addrspace(4)* + %src.bc = bitcast <4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8* + call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 32, i1 false) + %ref = load <4 x i64>, <4 x i64> addrspace(4)* %loc + ret <4 x i64> %ref +} + +define <1 x i64 addrspace(4)*> @forward_memcpy_vload3(<4 x i64 addrspace(4)*> addrspace(4)* %loc) { +; CHECK-LABEL: @forward_memcpy_vload3( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast <4 x i64 addrspace(4)*> addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)* +; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8*), i64 32, i1 false) +; CHECK-NEXT: ret <1 x i64 addrspace(4)*> +; +entry: + %loc.bc = bitcast <4 x i64 addrspace(4)*> addrspace(4)* %loc to i8 addrspace(4)* + %src.bc = bitcast <4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8* + call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 32, i1 false) + %ref = load <4 x i64 addrspace(4)*>, <4 x i64 addrspace(4)*> addrspace(4)* %loc + %val = extractelement <4 x i64 addrspace(4)*> %ref, i32 0 + %ret = insertelement <1 x i64 addrspace(4)*> undef, i64 addrspace(4)* %val, i32 0 + ret <1 x i64 addrspace(4)*> %ret +} ; Can forward since we can do so w/o breaking types -; TODO: missed optimization define i8 addrspace(4)* @forward_memcpy_zero(i8 addrspace(4)* addrspace(4)* %loc) { ; CHECK-LABEL: @forward_memcpy_zero( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)* ; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64>* @ZeroConstant to i8*), i64 8, i1 false) -; CHECK-NEXT: [[REF:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]] -; CHECK-NEXT: ret i8 addrspace(4)* [[REF]] +; CHECK-NEXT: ret i8 addrspace(4)* null ; entry: %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)* diff --git a/llvm/test/Transforms/GlobalOpt/evaluate-call-errors.ll b/llvm/test/Transforms/GlobalOpt/evaluate-call-errors.ll --- a/llvm/test/Transforms/GlobalOpt/evaluate-call-errors.ll +++ b/llvm/test/Transforms/GlobalOpt/evaluate-call-errors.ll @@ -65,7 +65,7 @@ } define internal %struct.Foo* @_ZL3foov() { - ret %struct.Foo* null + ret %struct.Foo* getelementptr (%struct.Foo, %struct.Foo *null, i32 1) } define linkonce_odr void @_ZN1QC2Ev(%struct.Q*) unnamed_addr align 2 { @@ -73,7 +73,7 @@ store %struct.Q* %0, %struct.Q** %2, align 8 %3 = load %struct.Q*, %struct.Q** %2, align 8 %4 = getelementptr inbounds %struct.Q, %struct.Q* %3, i32 0, i32 0 - %5 = call i32 bitcast (i32 (i32)* @_ZL3baz3Foo to i32 (%struct.Foo*)*)(%struct.Foo* null) + %5 = call i32 bitcast (i32 (i32)* @_ZL3baz3Foo to i32 (%struct.Foo*)*)(%struct.Foo* getelementptr (%struct.Foo, %struct.Foo *null, i32 1)) store i32 %5, i32* %4, align 4 ret void }