diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp --- a/llvm/lib/Analysis/ConstantFolding.cpp +++ b/llvm/lib/Analysis/ConstantFolding.cpp @@ -63,11 +63,6 @@ using namespace llvm; namespace { -Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP, - ArrayRef Ops, - const DataLayout &DL, - const TargetLibraryInfo *TLI, - bool ForLoadOperand); //===----------------------------------------------------------------------===// // Constant Folding internal helper functions @@ -550,19 +545,16 @@ return false; } -Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy, - const DataLayout &DL) { +Constant *FoldReinterpretLoadFromConst(Constant *C, Type *LoadTy, + int64_t Offset, const DataLayout &DL) { // Bail out early. Not expect to load from scalable global variable. if (isa(LoadTy)) return nullptr; - auto *PTy = cast(C->getType()); auto *IntType = dyn_cast(LoadTy); // If this isn't an integer load we can't fold it directly. if (!IntType) { - unsigned AS = PTy->getAddressSpace(); - // If this is a float/double load, we can try folding it as an int32/64 load // and then bitcast the result. This can be useful for union cases. Note // that address spaces don't matter here since we're not going to result in @@ -580,8 +572,7 @@ } else return nullptr; - C = FoldBitCast(C, MapTy->getPointerTo(AS), DL); - if (Constant *Res = FoldReinterpretLoadFromConstPtr(C, MapTy, DL)) { + if (Constant *Res = FoldReinterpretLoadFromConst(C, MapTy, Offset, DL)) { if (Res->isNullValue() && !LoadTy->isX86_MMXTy() && !LoadTy->isX86_AMXTy()) // Materializing a zero can be done trivially without a bitcast @@ -607,19 +598,7 @@ if (BytesLoaded > 32 || BytesLoaded == 0) return nullptr; - GlobalValue *GVal; - APInt OffsetAI; - if (!IsConstantOffsetFromGlobal(C, GVal, OffsetAI, DL)) - return nullptr; - - auto *GV = dyn_cast(GVal); - if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() || - !GV->getInitializer()->getType()->isSized()) - return nullptr; - - int64_t Offset = OffsetAI.getSExtValue(); - int64_t InitializerSize = - DL.getTypeAllocSize(GV->getInitializer()->getType()).getFixedSize(); + int64_t InitializerSize = DL.getTypeAllocSize(C->getType()).getFixedSize(); // If we're not accessing anything in this constant, the result is undefined. if (Offset <= -1 * static_cast(BytesLoaded)) @@ -640,7 +619,7 @@ Offset = 0; } - if (!ReadDataFromGlobal(GV->getInitializer(), Offset, CurPtr, BytesLeft, DL)) + if (!ReadDataFromGlobal(C, Offset, CurPtr, BytesLeft, DL)) return nullptr; APInt ResultVal = APInt(IntType->getBitWidth(), 0); @@ -661,111 +640,64 @@ return ConstantInt::get(IntType->getContext(), ResultVal); } -Constant *ConstantFoldLoadThroughBitcastExpr(ConstantExpr *CE, Type *DestTy, - const DataLayout &DL) { - auto *SrcPtr = CE->getOperand(0); - if (!SrcPtr->getType()->isPointerTy()) +/// If this Offset points exactly to the start of an aggregate element, return +/// that element, otherwise return nullptr. +Constant *getConstantAtOffset(Constant *Base, APInt Offset, + const DataLayout &DL) { + if (Offset.isZero()) + return Base; + + if (!isa(Base) && !isa(Base)) return nullptr; - return ConstantFoldLoadFromConstPtr(SrcPtr, DestTy, DL); + Type *ElemTy = Base->getType(); + SmallVector Indices = DL.getGEPIndicesForOffset(ElemTy, Offset); + if (!Offset.isZero() || !Indices[0].isZero()) + return nullptr; + + Constant *C = Base; + for (const APInt &Index : drop_begin(Indices)) { + if (Index.isNegative() || Index.getActiveBits() >= 32) + return nullptr; + + C = C->getAggregateElement(Index.getZExtValue()); + if (!C) + return nullptr; + } + + return C; +} + +Constant *ConstantFoldLoadFromConst(Constant *C, Type *Ty, const APInt &Offset, + const DataLayout &DL) { + if (Constant *AtOffset = getConstantAtOffset(C, Offset, DL)) + if (Constant *Result = ConstantFoldLoadThroughBitcast(AtOffset, Ty, DL)) + return Result; + + // Try hard to fold loads from bitcasted strange and non-type-safe things. + if (Offset.getMinSignedBits() <= 64) + return FoldReinterpretLoadFromConst(C, Ty, Offset.getSExtValue(), DL); + + return nullptr; } } // end anonymous namespace Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, const DataLayout &DL) { - // First, try the easy cases: + APInt Offset(DL.getIndexTypeSizeInBits(C->getType()), 0); + C = cast(C->stripAndAccumulateConstantOffsets( + DL, Offset, /* AllowNonInbounds */ true)); + if (auto *GV = dyn_cast(C)) if (GV->isConstant() && GV->hasDefinitiveInitializer()) - return ConstantFoldLoadThroughBitcast(GV->getInitializer(), Ty, DL); - - if (auto *GA = dyn_cast(C)) - if (GA->getAliasee() && !GA->isInterposable()) - return ConstantFoldLoadFromConstPtr(GA->getAliasee(), Ty, DL); - - // If the loaded value isn't a constant expr, we can't handle it. - auto *CE = dyn_cast(C); - if (!CE) - return nullptr; - - if (CE->getOpcode() == Instruction::GetElementPtr) { - if (auto *GV = dyn_cast(CE->getOperand(0))) { - if (GV->isConstant() && GV->hasDefinitiveInitializer()) { - if (Constant *V = ConstantFoldLoadThroughGEPConstantExpr( - GV->getInitializer(), CE, Ty, DL)) - return V; - } - } else { - // Try to simplify GEP if the pointer operand wasn't a GlobalVariable. - // SymbolicallyEvaluateGEP() with `ForLoadOperand = true` can potentially - // simplify the GEP more than it normally would have been, but should only - // be used for const folding loads. - SmallVector Ops; - for (unsigned I = 0, E = CE->getNumOperands(); I != E; ++I) - Ops.push_back(cast(CE->getOperand(I))); - if (auto *Simplified = dyn_cast_or_null( - SymbolicallyEvaluateGEP(cast(CE), Ops, DL, nullptr, - /*ForLoadOperand*/ true))) { - // If the symbolically evaluated GEP is another GEP, we can only const - // fold it if the resulting pointer operand is a GlobalValue. Otherwise - // there is nothing else to simplify since the GEP is already in the - // most simplified form. - if (isa(Simplified)) { - if (auto *GV = dyn_cast(Simplified->getOperand(0))) { - if (GV->isConstant() && GV->hasDefinitiveInitializer()) { - if (Constant *V = ConstantFoldLoadThroughGEPConstantExpr( - GV->getInitializer(), Simplified, Ty, DL)) - return V; - } - } - } else { - return ConstantFoldLoadFromConstPtr(Simplified, Ty, DL); - } - } - } - } - - if (CE->getOpcode() == Instruction::BitCast) - if (Constant *LoadedC = ConstantFoldLoadThroughBitcastExpr(CE, Ty, DL)) - return LoadedC; - - // Instead of loading constant c string, use corresponding integer value - // directly if string length is small enough. - StringRef Str; - if (getConstantStringInfo(CE, Str) && !Str.empty()) { - size_t StrLen = Str.size(); - unsigned NumBits = Ty->getPrimitiveSizeInBits(); - // Replace load with immediate integer if the result is an integer or fp - // value. - if ((NumBits >> 3) == StrLen + 1 && (NumBits & 7) == 0 && - (isa(Ty) || Ty->isFloatingPointTy())) { - APInt StrVal(NumBits, 0); - APInt SingleChar(NumBits, 0); - if (DL.isLittleEndian()) { - for (unsigned char C : reverse(Str.bytes())) { - SingleChar = static_cast(C); - StrVal = (StrVal << 8) | SingleChar; - } - } else { - for (unsigned char C : Str.bytes()) { - SingleChar = static_cast(C); - StrVal = (StrVal << 8) | SingleChar; - } - // Append NULL at the end. - SingleChar = 0; - StrVal = (StrVal << 8) | SingleChar; - } - - Constant *Res = ConstantInt::get(CE->getContext(), StrVal); - if (Ty->isFloatingPointTy()) - Res = ConstantExpr::getBitCast(Res, Ty); - return Res; - } - } + if (Constant *Result = ConstantFoldLoadFromConst(GV->getInitializer(), Ty, + Offset, DL)) + return Result; // If this load comes from anywhere in a constant global, and if the global // is all undef or zero, we know what it loads. - if (auto *GV = dyn_cast(getUnderlyingObject(CE))) { + if (auto *GV = dyn_cast(getUnderlyingObject(C))) { if (GV->isConstant() && GV->hasDefinitiveInitializer()) { if (GV->getInitializer()->isNullValue()) return Constant::getNullValue(Ty); @@ -774,8 +706,7 @@ } } - // Try hard to fold loads from bitcasted strange and non-type-safe things. - return FoldReinterpretLoadFromConstPtr(CE, Ty, DL); + return nullptr; } namespace { @@ -867,17 +798,10 @@ } /// Strip the pointer casts, but preserve the address space information. -Constant *StripPtrCastKeepAS(Constant *Ptr, bool ForLoadOperand) { +Constant *StripPtrCastKeepAS(Constant *Ptr) { assert(Ptr->getType()->isPointerTy() && "Not a pointer type"); auto *OldPtrTy = cast(Ptr->getType()); Ptr = cast(Ptr->stripPointerCasts()); - if (ForLoadOperand) { - while (isa(Ptr) && !cast(Ptr)->isInterposable() && - !cast(Ptr)->getBaseObject()->isInterposable()) { - Ptr = cast(Ptr)->getAliasee(); - } - } - auto *NewPtrTy = cast(Ptr->getType()); // Preserve the address space number of the pointer. @@ -893,8 +817,7 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP, ArrayRef Ops, const DataLayout &DL, - const TargetLibraryInfo *TLI, - bool ForLoadOperand) { + const TargetLibraryInfo *TLI) { const GEPOperator *InnermostGEP = GEP; bool InBounds = GEP->isInBounds(); @@ -939,7 +862,7 @@ DL.getIndexedOffsetInType( SrcElemTy, makeArrayRef((Value * const *)Ops.data() + 1, Ops.size() - 1))); - Ptr = StripPtrCastKeepAS(Ptr, ForLoadOperand); + Ptr = StripPtrCastKeepAS(Ptr); // If this is a GEP of a GEP, fold it all into a single GEP. while (auto *GEP = dyn_cast(Ptr)) { @@ -961,7 +884,7 @@ Ptr = cast(GEP->getOperand(0)); SrcElemTy = GEP->getSourceElementType(); Offset += APInt(BitWidth, DL.getIndexedOffsetInType(SrcElemTy, NestedOps)); - Ptr = StripPtrCastKeepAS(Ptr, ForLoadOperand); + Ptr = StripPtrCastKeepAS(Ptr); } // If the base value for this address is a literal integer value, fold the @@ -1069,8 +992,7 @@ return ConstantFoldCastOperand(Opcode, Ops[0], DestTy, DL); if (auto *GEP = dyn_cast(InstOrCE)) { - if (Constant *C = SymbolicallyEvaluateGEP(GEP, Ops, DL, TLI, - /*ForLoadOperand*/ false)) + if (Constant *C = SymbolicallyEvaluateGEP(GEP, Ops, DL, TLI)) return C; return ConstantExpr::getGetElementPtr(GEP->getSourceElementType(), Ops[0], diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/loads.ll b/llvm/test/Transforms/InstSimplify/ConstProp/loads.ll --- a/llvm/test/Transforms/InstSimplify/ConstProp/loads.ll +++ b/llvm/test/Transforms/InstSimplify/ConstProp/loads.ll @@ -30,11 +30,12 @@ ret i16 %r } -; FIXME: Should be able to load through a constant addrspacecast. define i16 @test2_addrspacecast() { -; CHECK-LABEL: @test2_addrspacecast( -; CHECK-NEXT: [[R:%.*]] = load i16, i16 addrspace(1)* addrspacecast (i16* bitcast ({ { i32, i8 }, i32 }* @g1 to i16*) to i16 addrspace(1)*), align 8 -; CHECK-NEXT: ret i16 [[R]] +; LE-LABEL: @test2_addrspacecast( +; LE-NEXT: ret i16 -16657 +; +; BE-LABEL: @test2_addrspacecast( +; BE-NEXT: ret i16 -8531 ; %r = load i16, i16 addrspace(1)* addrspacecast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16 addrspace(1)*) ret i16 %r @@ -246,7 +247,7 @@ define i64 @test_array_of_zero_size_array() { ; CHECK-LABEL: @test_array_of_zero_size_array( -; CHECK-NEXT: ret i64 0 +; CHECK-NEXT: ret i64 undef ; %v = load i64, i64* bitcast ([4294967295 x [0 x i32]]* @g9 to i64*) ret i64 %v @@ -266,8 +267,7 @@ define {}* @test_trailing_zero_gep_index() { ; CHECK-LABEL: @test_trailing_zero_gep_index( -; CHECK-NEXT: [[V:%.*]] = load {}*, {}** bitcast (i8* getelementptr inbounds (<{ [8 x i8], [8 x i8] }>, <{ [8 x i8], [8 x i8] }>* @g11, i64 0, i32 1, i64 0) to {}**), align 4 -; CHECK-NEXT: ret {}* [[V]] +; CHECK-NEXT: ret {}* null ; %v = load {}*, {}** bitcast (i8* getelementptr inbounds (<{ [8 x i8], [8 x i8] }>, <{ [8 x i8], [8 x i8] }>* @g11, i32 0, i32 1, i32 0) to {}**), align 4 ret {}* %v diff --git a/llvm/test/Transforms/InstSimplify/load.ll b/llvm/test/Transforms/InstSimplify/load.ll --- a/llvm/test/Transforms/InstSimplify/load.ll +++ b/llvm/test/Transforms/InstSimplify/load.ll @@ -6,7 +6,7 @@ define i32 @crash_on_zeroinit() { ; CHECK-LABEL: @crash_on_zeroinit( -; CHECK-NEXT: ret i32 0 +; CHECK-NEXT: ret i32 undef ; %load = load i32, i32* bitcast ({}* @zeroinit to i32*) ret i32 %load