diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h --- a/llvm/include/llvm/Analysis/ConstantFolding.h +++ b/llvm/include/llvm/Analysis/ConstantFolding.h @@ -28,10 +28,12 @@ class DSOLocalEquivalent; class DataLayout; class Function; +class GEPOperator; class GlobalValue; class Instruction; class TargetLibraryInfo; class Type; +class Value; /// If this constant is a constant offset from a global, return the global and /// the constant. Because of constantexprs, this function is recursive. @@ -128,17 +130,16 @@ Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2, ArrayRef Mask); -/// ConstantFoldLoadFromConstPtr - Return the value that a load from C would +/// ConstantFoldLoadFromValue - Return the value that a load from V would /// produce if it is constant and determinable. If this is not determinable, /// return null. -Constant *ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, const DataLayout &DL); +Constant *ConstantFoldLoadFromValue(Value *V, Type *Ty, const DataLayout &DL); -/// ConstantFoldLoadThroughGEPConstantExpr - Given a constant and a -/// getelementptr constantexpr, return the constant value being addressed by the +/// ConstantFoldLoadThroughGEP - Given a constant and a +/// getelementptr, return the constant value being addressed by the /// constant expression, or null if something is funny and we can't decide. -Constant *ConstantFoldLoadThroughGEPConstantExpr(Constant *C, ConstantExpr *CE, - Type *Ty, - const DataLayout &DL); +Constant *ConstantFoldLoadThroughGEP(Constant *C, GEPOperator *GEP, Type *Ty, + const DataLayout &DL); /// ConstantFoldLoadThroughGEPIndices - Given a constant and getelementptr /// indices (with an *implied* zero pointer index that is not in the list), diff --git a/llvm/include/llvm/IR/Value.h b/llvm/include/llvm/IR/Value.h --- a/llvm/include/llvm/IR/Value.h +++ b/llvm/include/llvm/IR/Value.h @@ -643,6 +643,18 @@ static_cast(this)->stripPointerCasts()); } + /// Strip off pointer casts, local aliases to local aliasees, and invariant + /// group info. + /// + /// Returns the original uncasted value. If this is called on a non-pointer + /// value, it returns 'this'. This function should be used only in constant + /// folding loads. + const Value *stripPointerCastsForLoadOperand() const; + Value *stripPointerCastsForLoadOperand() { + return const_cast( + static_cast(this)->stripPointerCastsForLoadOperand()); + } + /// Strip off pointer casts, all-zero GEPs, address space casts, and aliases. /// /// Returns the original uncasted value. If this is called on a non-pointer diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp --- a/llvm/lib/Analysis/ConstantFolding.cpp +++ b/llvm/lib/Analysis/ConstantFolding.cpp @@ -63,6 +63,11 @@ using namespace llvm; namespace { +Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP, Value *Ptr, + ArrayRef Idxs, + const DataLayout &DL, + const TargetLibraryInfo *TLI, + bool ForLoadOperand); //===----------------------------------------------------------------------===// // Constant Folding internal helper functions @@ -656,51 +661,74 @@ return ConstantInt::get(IntType->getContext(), ResultVal); } -Constant *ConstantFoldLoadThroughBitcastExpr(ConstantExpr *CE, Type *DestTy, +Constant *ConstantFoldLoadThroughBitcastExpr(BitCastOperator *BC, Type *DestTy, const DataLayout &DL) { - auto *SrcPtr = CE->getOperand(0); + auto *SrcPtr = BC->getOperand(0); if (!SrcPtr->getType()->isPointerTy()) return nullptr; - return ConstantFoldLoadFromConstPtr(SrcPtr, DestTy, DL); + return ConstantFoldLoadFromValue(SrcPtr, DestTy, DL); } } // end anonymous namespace -Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, - const DataLayout &DL) { +Constant *llvm::ConstantFoldLoadFromValue(Value *V, Type *Ty, + const DataLayout &DL) { // First, try the easy cases: - if (auto *GV = dyn_cast(C)) + if (auto *GV = dyn_cast(V)) if (GV->isConstant() && GV->hasDefinitiveInitializer()) return ConstantFoldLoadThroughBitcast(GV->getInitializer(), Ty, DL); - if (auto *GA = dyn_cast(C)) + if (auto *GA = dyn_cast(V)) if (GA->getAliasee() && !GA->isInterposable()) - return ConstantFoldLoadFromConstPtr(GA->getAliasee(), Ty, DL); + return ConstantFoldLoadFromValue(GA->getAliasee(), Ty, DL); - // If the loaded value isn't a constant expr, we can't handle it. - auto *CE = dyn_cast(C); - if (!CE) - return nullptr; - - if (CE->getOpcode() == Instruction::GetElementPtr) { - if (auto *GV = dyn_cast(CE->getOperand(0))) { - if (GV->isConstant() && GV->hasDefinitiveInitializer()) { - if (Constant *V = ConstantFoldLoadThroughGEPConstantExpr( - GV->getInitializer(), CE, Ty, DL)) - return V; + if (auto *GEP = dyn_cast(V)) { + SmallVector Idxs; + for (unsigned I = 1, E = GEP->getNumOperands(); I != E; ++I) { + if (auto *Idx = dyn_cast(GEP->getOperand(I))) + Idxs.push_back(Idx); + else + return nullptr; + } + if (auto *Simplified = SymbolicallyEvaluateGEP( + GEP, GEP->getPointerOperand(), Idxs, DL, nullptr, + /*ForLoadOperand*/ true)) { + if (auto *SimplifiedGEP = dyn_cast(Simplified)) { + if (auto *GV = + dyn_cast(SimplifiedGEP->getPointerOperand())) { + if (GV->isConstant() && GV->hasDefinitiveInitializer()) { + if (Constant *V = ConstantFoldLoadThroughGEP( + GV->getInitializer(), SimplifiedGEP, Ty, DL)) { + assert(V->getType() == Ty && "wrong type"); + return V; + } + } + } + } else { + return ConstantFoldLoadFromValue(Simplified, Ty, DL); } } } - if (CE->getOpcode() == Instruction::BitCast) - if (Constant *LoadedC = ConstantFoldLoadThroughBitcastExpr(CE, Ty, DL)) + if (auto *BC = dyn_cast(V)) { + if (Constant *LoadedC = ConstantFoldLoadThroughBitcastExpr(BC, Ty, DL)) { + assert(LoadedC->getType() == Ty && "wrong type"); return LoadedC; + } + } + + if (auto *Call = dyn_cast(V)) { + if (Call->getIntrinsicID() == Intrinsic::strip_invariant_group || + Call->getIntrinsicID() == Intrinsic::launder_invariant_group) { + return ConstantFoldLoadFromValue(Call->getOperand(0), Ty, DL); + } + } // Instead of loading constant c string, use corresponding integer value // directly if string length is small enough. StringRef Str; - if (getConstantStringInfo(CE, Str) && !Str.empty()) { + if (getConstantStringInfo(V, Str) && !Str.empty()) { size_t StrLen = Str.size(); unsigned NumBits = Ty->getPrimitiveSizeInBits(); // Replace load with immediate integer if the result is an integer or fp @@ -724,7 +752,7 @@ StrVal = (StrVal << 8) | SingleChar; } - Constant *Res = ConstantInt::get(CE->getContext(), StrVal); + Constant *Res = ConstantInt::get(V->getContext(), StrVal); if (Ty->isFloatingPointTy()) Res = ConstantExpr::getBitCast(Res, Ty); return Res; @@ -733,7 +761,7 @@ // If this load comes from anywhere in a constant global, and if the global // is all undef or zero, we know what it loads. - if (auto *GV = dyn_cast(getUnderlyingObject(CE))) { + if (auto *GV = dyn_cast(getUnderlyingObject(V))) { if (GV->isConstant() && GV->hasDefinitiveInitializer()) { if (GV->getInitializer()->isNullValue()) return Constant::getNullValue(Ty); @@ -743,7 +771,10 @@ } // Try hard to fold loads from bitcasted strange and non-type-safe things. - return FoldReinterpretLoadFromConstPtr(CE, Ty, DL); + if (auto *C = dyn_cast(V)) + return FoldReinterpretLoadFromConstPtr(C, Ty, DL); + + return nullptr; } namespace { @@ -751,10 +782,7 @@ Constant *ConstantFoldLoadInst(const LoadInst *LI, const DataLayout &DL) { if (LI->isVolatile()) return nullptr; - if (auto *C = dyn_cast(LI->getOperand(0))) - return ConstantFoldLoadFromConstPtr(C, LI->getType(), DL); - - return nullptr; + return ConstantFoldLoadFromValue(LI->getOperand(0), LI->getType(), DL); } /// One of Op0/Op1 is a constant expression. @@ -809,62 +837,66 @@ /// If array indices are not pointer-sized integers, explicitly cast them so /// that they aren't implicitly casted by the getelementptr. -Constant *CastGEPIndices(Type *SrcElemTy, ArrayRef Ops, - Type *ResultTy, Optional InRangeIndex, - const DataLayout &DL, const TargetLibraryInfo *TLI) { +Constant *CastGEPIndices(Type *SrcElemTy, Constant *Ptr, + ArrayRef Idxs, Type *ResultTy, + Optional InRangeIndex, const DataLayout &DL, + const TargetLibraryInfo *TLI) { Type *IntIdxTy = DL.getIndexType(ResultTy); Type *IntIdxScalarTy = IntIdxTy->getScalarType(); bool Any = false; SmallVector NewIdxs; - for (unsigned i = 1, e = Ops.size(); i != e; ++i) { - if ((i == 1 || - !isa(GetElementPtrInst::getIndexedType( - SrcElemTy, Ops.slice(1, i - 1)))) && - Ops[i]->getType()->getScalarType() != IntIdxScalarTy) { + for (unsigned i = 0, e = Idxs.size(); i != e; ++i) { + if ((i == 0 || !isa(GetElementPtrInst::getIndexedType( + SrcElemTy, Idxs.slice(0, i)))) && + Idxs[i]->getType()->getScalarType() != IntIdxScalarTy) { Any = true; - Type *NewType = Ops[i]->getType()->isVectorTy() - ? IntIdxTy - : IntIdxScalarTy; - NewIdxs.push_back(ConstantExpr::getCast(CastInst::getCastOpcode(Ops[i], - true, - NewType, - true), - Ops[i], NewType)); + Type *NewType = + Idxs[i]->getType()->isVectorTy() ? IntIdxTy : IntIdxScalarTy; + NewIdxs.push_back(ConstantExpr::getCast( + CastInst::getCastOpcode(Idxs[i], true, NewType, true), Idxs[i], + NewType)); } else - NewIdxs.push_back(Ops[i]); + NewIdxs.push_back(Idxs[i]); } if (!Any) return nullptr; Constant *C = ConstantExpr::getGetElementPtr( - SrcElemTy, Ops[0], NewIdxs, /*InBounds=*/false, InRangeIndex); + SrcElemTy, Ptr, NewIdxs, /*InBounds=*/false, InRangeIndex); return ConstantFoldConstant(C, DL, TLI); } -/// Strip the pointer casts, but preserve the address space information. -Constant *StripPtrCastKeepAS(Constant *Ptr, Type *&ElemTy) { +/// Strip pointer casts, but preserve address space information. +Value *StripPtrCastKeepAS(Value *Ptr, Type *&ElemTy, bool ForLoadOperand) { assert(Ptr->getType()->isPointerTy() && "Not a pointer type"); auto *OldPtrTy = cast(Ptr->getType()); - Ptr = cast(Ptr->stripPointerCasts()); - auto *NewPtrTy = cast(Ptr->getType()); + Constant *NewPtr; + if (ForLoadOperand) + NewPtr = dyn_cast(Ptr->stripPointerCastsForLoadOperand()); + else + NewPtr = dyn_cast(Ptr->stripPointerCasts()); + if (!NewPtr) + return Ptr; + auto *NewPtrTy = cast(NewPtr->getType()); ElemTy = NewPtrTy->getPointerElementType(); // Preserve the address space number of the pointer. if (NewPtrTy->getAddressSpace() != OldPtrTy->getAddressSpace()) { NewPtrTy = ElemTy->getPointerTo(OldPtrTy->getAddressSpace()); - Ptr = ConstantExpr::getPointerCast(Ptr, NewPtrTy); + NewPtr = ConstantExpr::getPointerCast(NewPtr, NewPtrTy); } - return Ptr; + return NewPtr; } /// If we can symbolically evaluate the GEP constant expression, do so. -Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP, - ArrayRef Ops, +Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP, Value *Ptr, + ArrayRef Idxs, const DataLayout &DL, - const TargetLibraryInfo *TLI) { + const TargetLibraryInfo *TLI, + bool ForLoadOperand) { const GEPOperator *InnermostGEP = GEP; bool InBounds = GEP->isInBounds(); @@ -874,11 +906,12 @@ if (!SrcElemTy->isSized() || isa(SrcElemTy)) return nullptr; - if (Constant *C = CastGEPIndices(SrcElemTy, Ops, ResTy, - GEP->getInRangeIndex(), DL, TLI)) - return C; + if (Constant *PtrC = dyn_cast(Ptr)) { + if (Constant *C = CastGEPIndices(SrcElemTy, PtrC, Idxs, ResTy, + GEP->getInRangeIndex(), DL, TLI)) + return C; + } - Constant *Ptr = Ops[0]; if (!Ptr->getType()->isPointerTy()) return nullptr; @@ -886,33 +919,34 @@ // If this is a constant expr gep that is effectively computing an // "offsetof", fold it into 'cast int Size to T*' instead of 'gep 0, 0, 12' - for (unsigned i = 1, e = Ops.size(); i != e; ++i) - if (!isa(Ops[i])) { - - // If this is "gep i8* Ptr, (sub 0, V)", fold this as: - // "inttoptr (sub (ptrtoint Ptr), V)" - if (Ops.size() == 2 && ResElemTy->isIntegerTy(8)) { - auto *CE = dyn_cast(Ops[1]); - assert((!CE || CE->getType() == IntIdxTy) && - "CastGEPIndices didn't canonicalize index types!"); - if (CE && CE->getOpcode() == Instruction::Sub && - CE->getOperand(0)->isNullValue()) { - Constant *Res = ConstantExpr::getPtrToInt(Ptr, CE->getType()); + for (unsigned i = 0, e = Idxs.size(); i != e; ++i) { + if (!isa(Idxs[i])) { + // If this is "gep i8* Ptr, (sub 0, V)", fold this as: + // "inttoptr (sub (ptrtoint Ptr), V)" + if (Idxs.size() == 1 && ResElemTy->isIntegerTy(8)) { + auto *CE = dyn_cast(Idxs[0]); + assert((!CE || CE->getType() == IntIdxTy) && + "CastGEPIndices didn't canonicalize index types!"); + if (CE && CE->getOpcode() == Instruction::Sub && + CE->getOperand(0)->isNullValue()) { + if (auto *PtrC = dyn_cast(Ptr)) { + Constant *Res = ConstantExpr::getPtrToInt(PtrC, CE->getType()); Res = ConstantExpr::getSub(Res, CE->getOperand(1)); Res = ConstantExpr::getIntToPtr(Res, ResTy); return ConstantFoldConstant(Res, DL, TLI); } } - return nullptr; } + return nullptr; + } + } unsigned BitWidth = DL.getTypeSizeInBits(IntIdxTy); - APInt Offset = - APInt(BitWidth, - DL.getIndexedOffsetInType( - SrcElemTy, - makeArrayRef((Value * const *)Ops.data() + 1, Ops.size() - 1))); - Ptr = StripPtrCastKeepAS(Ptr, SrcElemTy); + APInt Offset = APInt( + BitWidth, + DL.getIndexedOffsetInType( + SrcElemTy, makeArrayRef((Value *const *)Idxs.begin(), Idxs.size()))); + Ptr = StripPtrCastKeepAS(Ptr, SrcElemTy, ForLoadOperand); // If this is a GEP of a GEP, fold it all into a single GEP. while (auto *GEP = dyn_cast(Ptr)) { @@ -931,10 +965,12 @@ if (!AllConstantInt) break; - Ptr = cast(GEP->getOperand(0)); + Ptr = dyn_cast(GEP->getOperand(0)); + if (!Ptr) + return nullptr; SrcElemTy = GEP->getSourceElementType(); Offset += APInt(BitWidth, DL.getIndexedOffsetInType(SrcElemTy, NestedOps)); - Ptr = StripPtrCastKeepAS(Ptr, SrcElemTy); + Ptr = StripPtrCastKeepAS(Ptr, SrcElemTy, ForLoadOperand); } // If the base value for this address is a literal integer value, fold the @@ -948,9 +984,12 @@ } auto *PTy = cast(Ptr->getType()); - if ((Ptr->isNullValue() || BasePtr != 0) && + auto *PtrC = dyn_cast(Ptr); + if (!PtrC) + return nullptr; + if ((PtrC->isNullValue() || BasePtr != 0) && !DL.isNonIntegralPointerType(PTy)) { - Constant *C = ConstantInt::get(Ptr->getContext(), Offset + BasePtr); + Constant *C = ConstantInt::get(PtrC->getContext(), Offset + BasePtr); return ConstantExpr::getIntToPtr(C, ResTy); } @@ -1037,7 +1076,7 @@ } // Create a GEP. - Constant *C = ConstantExpr::getGetElementPtr(SrcElemTy, Ptr, NewIdxs, + Constant *C = ConstantExpr::getGetElementPtr(SrcElemTy, PtrC, NewIdxs, InBounds, InRangeIndex); assert(C->getType()->getPointerElementType() == Ty && "Computed GetElementPtr has unexpected type!"); @@ -1071,7 +1110,9 @@ return ConstantFoldCastOperand(Opcode, Ops[0], DestTy, DL); if (auto *GEP = dyn_cast(InstOrCE)) { - if (Constant *C = SymbolicallyEvaluateGEP(GEP, Ops, DL, TLI)) + if (Constant *C = + SymbolicallyEvaluateGEP(GEP, Ops[0], Ops.slice(1), DL, TLI, + /*ForLoadOperand*/ false)) return C; return ConstantExpr::getGetElementPtr(GEP->getSourceElementType(), Ops[0], @@ -1188,6 +1229,9 @@ return CommonValue ? CommonValue : UndefValue::get(PN->getType()); } + if (const auto *LI = dyn_cast(I)) + return ConstantFoldLoadInst(LI, DL); + // Scan the operand list, checking to see if they are all constants, if so, // hand off to ConstantFoldInstOperandsImpl. if (!all_of(I->operands(), [](Use &U) { return isa(U); })) @@ -1206,9 +1250,6 @@ return ConstantFoldCompareInstOperands(CI->getPredicate(), Ops[0], Ops[1], DL, TLI); - if (const auto *LI = dyn_cast(I)) - return ConstantFoldLoadInst(LI, DL); - if (auto *IVI = dyn_cast(I)) { return ConstantExpr::getInsertValue( cast(IVI->getAggregateOperand()), @@ -1404,17 +1445,19 @@ } } -Constant *llvm::ConstantFoldLoadThroughGEPConstantExpr(Constant *C, - ConstantExpr *CE, - Type *Ty, - const DataLayout &DL) { - if (!CE->getOperand(1)->isNullValue()) +Constant *llvm::ConstantFoldLoadThroughGEP(Constant *C, GEPOperator *GEP, + Type *Ty, const DataLayout &DL) { + auto *O = dyn_cast(GEP->getOperand(1)); + if (!O || !O->isNullValue()) return nullptr; // Do not allow stepping over the value! // Loop over all of the operands, tracking down which value we are // addressing. - for (unsigned i = 2, e = CE->getNumOperands(); i != e; ++i) { - C = C->getAggregateElement(CE->getOperand(i)); + for (unsigned I = 2, E = GEP->getNumOperands(); I != E; ++I) { + auto *Idx = dyn_cast(GEP->getOperand(I)); + if (!Idx) + return nullptr; + C = C->getAggregateElement(Idx); if (!C) return nullptr; } @@ -2863,7 +2906,7 @@ auto *Mask = Operands[2]; auto *Passthru = Operands[3]; - Constant *VecData = ConstantFoldLoadFromConstPtr(SrcPtr, FVTy, DL); + Constant *VecData = ConstantFoldLoadFromValue(SrcPtr, FVTy, DL); SmallVector NewElements; for (unsigned I = 0, E = FVTy->getNumElements(); I != E; ++I) { diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -4026,7 +4026,7 @@ if (LoadInst *LI = dyn_cast(I)) if (!LI->isVolatile()) - return ConstantFoldLoadFromConstPtr(ConstOps[0], LI->getType(), Q.DL); + return ConstantFoldLoadFromValue(ConstOps[0], LI->getType(), Q.DL); return ConstantFoldInstOperands(I, ConstOps, Q.DL, Q.TLI); } @@ -5288,7 +5288,7 @@ Constant *C = ConstantExpr::getGetElementPtr( Int32Ty, ConstantExpr::getBitCast(Ptr, Int32PtrTy), ConstantInt::get(Int64Ty, OffsetInt / 4)); - Constant *Loaded = ConstantFoldLoadFromConstPtr(C, Int32Ty, DL); + Constant *Loaded = ConstantFoldLoadFromValue(C, Int32Ty, DL); if (!Loaded) return nullptr; diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -8251,7 +8251,7 @@ Operands[1], DL, TLI); if (LoadInst *LI = dyn_cast(I)) { if (!LI->isVolatile()) - return ConstantFoldLoadFromConstPtr(Operands[0], LI->getType(), DL); + return ConstantFoldLoadFromValue(Operands[0], LI->getType(), DL); } return ConstantFoldInstOperands(I, Operands, DL, TLI); } @@ -8675,8 +8675,7 @@ Operands[1], DL, &TLI); else if (const LoadInst *Load = dyn_cast(I)) { if (!Load->isVolatile()) - C = ConstantFoldLoadFromConstPtr(Operands[0], Load->getType(), - DL); + C = ConstantFoldLoadFromValue(Operands[0], Load->getType(), DL); } else C = ConstantFoldInstOperands(I, Operands, DL, &TLI); if (!C) return V; diff --git a/llvm/lib/CodeGen/ExpandMemCmp.cpp b/llvm/lib/CodeGen/ExpandMemCmp.cpp --- a/llvm/lib/CodeGen/ExpandMemCmp.cpp +++ b/llvm/lib/CodeGen/ExpandMemCmp.cpp @@ -298,15 +298,11 @@ RhsSource = Builder.CreateBitCast(RhsSource, LoadSizeType->getPointerTo()); // Create a constant or a load from the source. - Value *Lhs = nullptr; - if (auto *C = dyn_cast(LhsSource)) - Lhs = ConstantFoldLoadFromConstPtr(C, LoadSizeType, DL); + Value *Lhs = ConstantFoldLoadFromValue(LhsSource, LoadSizeType, DL); if (!Lhs) Lhs = Builder.CreateAlignedLoad(LoadSizeType, LhsSource, LhsAlign); - Value *Rhs = nullptr; - if (auto *C = dyn_cast(RhsSource)) - Rhs = ConstantFoldLoadFromConstPtr(C, LoadSizeType, DL); + Value *Rhs = ConstantFoldLoadFromValue(RhsSource, LoadSizeType, DL); if (!Rhs) Rhs = Builder.CreateAlignedLoad(LoadSizeType, RhsSource, RhsAlign); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -7469,7 +7469,7 @@ LoadInput = ConstantExpr::getBitCast(const_cast(LoadInput), PointerType::getUnqual(LoadTy)); - if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr( + if (const Constant *LoadCst = ConstantFoldLoadFromValue( const_cast(LoadInput), LoadTy, *Builder.DL)) return Builder.getValue(LoadCst); } diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp --- a/llvm/lib/IR/Value.cpp +++ b/llvm/lib/IR/Value.cpp @@ -568,6 +568,7 @@ PSK_ZeroIndicesAndAliases, PSK_ZeroIndicesSameRepresentation, PSK_ForAliasAnalysis, + PSK_ForLoadOperand, PSK_InBoundsConstantIndices, PSK_InBounds }; @@ -605,6 +606,8 @@ if (!GEP->isInBounds()) return V; break; + case PSK_ForLoadOperand: + return V; } V = GEP->getPointerOperand(); } else if (Operator::getOpcode(V) == Instruction::BitCast) { @@ -618,6 +621,8 @@ V = cast(V)->getOperand(0); } else if (StripKind == PSK_ZeroIndicesAndAliases && isa(V)) { V = cast(V)->getAliasee(); + } else if (StripKind == PSK_ForLoadOperand && isa(V)) { + V = cast(V)->getAliasee(); } else if (StripKind == PSK_ForAliasAnalysis && isa(V) && cast(V)->getNumIncomingValues() == 1) { V = cast(V)->getIncomingValue(0); @@ -630,7 +635,8 @@ // The result of launder.invariant.group must alias it's argument, // but it can't be marked with returned attribute, that's why it needs // special case. - if (StripKind == PSK_ForAliasAnalysis && + if ((StripKind == PSK_ForAliasAnalysis || + StripKind == PSK_ForLoadOperand) && (Call->getIntrinsicID() == Intrinsic::launder_invariant_group || Call->getIntrinsicID() == Intrinsic::strip_invariant_group)) { V = Call->getArgOperand(0); @@ -649,6 +655,9 @@ const Value *Value::stripPointerCasts() const { return stripPointerCastsAndOffsets(this); } +const Value *Value::stripPointerCastsForLoadOperand() const { + return stripPointerCastsAndOffsets(this); +} const Value *Value::stripPointerCastsAndAliases() const { return stripPointerCastsAndOffsets(this); diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp --- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp +++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp @@ -309,11 +309,11 @@ SI->eraseFromParent(); Changed = true; } else if (ConstantExpr *CE = dyn_cast(U)) { - if (CE->getOpcode() == Instruction::GetElementPtr) { + if (auto *GEP = dyn_cast(CE)) { Constant *SubInit = nullptr; if (Init) - SubInit = ConstantFoldLoadThroughGEPConstantExpr( - Init, CE, V->getType()->getPointerElementType(), DL); + SubInit = ConstantFoldLoadThroughGEP( + Init, GEP, V->getType()->getPointerElementType(), DL); Changed |= CleanupConstantGlobalUsers(CE, SubInit, DL, GetTLI); } else if ((CE->getOpcode() == Instruction::BitCast && CE->getType()->isPointerTy()) || @@ -335,8 +335,9 @@ ConstantExpr *CE = dyn_cast_or_null( ConstantFoldInstruction(GEP, DL, &GetTLI(*GEP->getFunction()))); if (Init && CE && CE->getOpcode() == Instruction::GetElementPtr) - SubInit = ConstantFoldLoadThroughGEPConstantExpr( - Init, CE, V->getType()->getPointerElementType(), DL); + SubInit = ConstantFoldLoadThroughGEP( + Init, cast(GEP), + V->getType()->getPointerElementType(), DL); // If the initializer is an all-null value and we have an inbounds GEP, // we already know what the result of any load from that GEP is. diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -3660,8 +3660,7 @@ continue; // Instruction isn't dead, see if we can constant propagate it. - if (!I->use_empty() && - (I->getNumOperands() == 0 || isa(I->getOperand(0)))) { + if (!I->use_empty()) { if (Constant *C = ConstantFoldInstruction(I, DL, &TLI)) { LLVM_DEBUG(dbgs() << "IC: ConstFold to: " << *C << " from: " << *I << '\n'); @@ -3871,8 +3870,7 @@ Instruction *Inst = &*BBI++; // ConstantProp instruction if trivially constant. - if (!Inst->use_empty() && - (Inst->getNumOperands() == 0 || isa(Inst->getOperand(0)))) + if (!Inst->use_empty()) if (Constant *C = ConstantFoldInstruction(Inst, DL, TLI)) { LLVM_DEBUG(dbgs() << "IC: ConstFold to: " << *C << " from: " << *Inst << '\n'); diff --git a/llvm/lib/Transforms/Utils/Evaluator.cpp b/llvm/lib/Transforms/Utils/Evaluator.cpp --- a/llvm/lib/Transforms/Utils/Evaluator.cpp +++ b/llvm/lib/Transforms/Utils/Evaluator.cpp @@ -157,8 +157,8 @@ if (!CE->isGEPWithNoNotionalOverIndexing()) return false; - return ConstantFoldLoadThroughGEPConstantExpr( - GV->getInitializer(), CE, + return ConstantFoldLoadThroughGEP( + GV->getInitializer(), cast(CE), cast(CE)->getResultElementType(), DL); } else if (CE->getOpcode() == Instruction::BitCast && isa(CE->getOperand(0))) { @@ -228,7 +228,7 @@ // Handle a constantexpr getelementptr. case Instruction::GetElementPtr: if (auto *I = getInitializer(CE->getOperand(0))) - return ConstantFoldLoadThroughGEPConstantExpr(I, CE, Ty, DL); + return ConstantFoldLoadThroughGEP(I, cast(CE), Ty, DL); break; // Handle a constantexpr bitcast. case Instruction::BitCast: diff --git a/llvm/lib/Transforms/Utils/SCCPSolver.cpp b/llvm/lib/Transforms/Utils/SCCPSolver.cpp --- a/llvm/lib/Transforms/Utils/SCCPSolver.cpp +++ b/llvm/lib/Transforms/Utils/SCCPSolver.cpp @@ -1122,7 +1122,7 @@ } // Transform load from a constant into a constant if possible. - if (Constant *C = ConstantFoldLoadFromConstPtr(Ptr, I.getType(), DL)) { + if (Constant *C = ConstantFoldLoadFromValue(Ptr, I.getType(), DL)) { if (isa(C)) return; return (void)markConstant(IV, &I, C); diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp --- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp @@ -974,16 +974,8 @@ unsigned PrefAlignment = DL.getPrefTypeAlignment(IntType); // First, see if we can fold either argument to a constant. - Value *LHSV = nullptr; - if (auto *LHSC = dyn_cast(LHS)) { - LHSC = ConstantExpr::getBitCast(LHSC, IntType->getPointerTo()); - LHSV = ConstantFoldLoadFromConstPtr(LHSC, IntType, DL); - } - Value *RHSV = nullptr; - if (auto *RHSC = dyn_cast(RHS)) { - RHSC = ConstantExpr::getBitCast(RHSC, IntType->getPointerTo()); - RHSV = ConstantFoldLoadFromConstPtr(RHSC, IntType, DL); - } + Value *LHSV = ConstantFoldLoadFromValue(LHS, IntType, DL); + Value *RHSV = ConstantFoldLoadFromValue(RHS, IntType, DL); // Don't generate unaligned loads. If either source is constant data, // alignment doesn't matter for that source because there is no load. diff --git a/llvm/lib/Transforms/Utils/VNCoercion.cpp b/llvm/lib/Transforms/Utils/VNCoercion.cpp --- a/llvm/lib/Transforms/Utils/VNCoercion.cpp +++ b/llvm/lib/Transforms/Utils/VNCoercion.cpp @@ -415,7 +415,7 @@ Src, OffsetCst); } Src = ConstantExpr::getBitCast(Src, PointerType::get(LoadTy, AS)); - if (ConstantFoldLoadFromConstPtr(Src, LoadTy, DL)) + if (ConstantFoldLoadFromValue(Src, LoadTy, DL)) return Offset; return -1; } @@ -596,7 +596,7 @@ Src, OffsetCst); } Src = ConstantExpr::getBitCast(Src, PointerType::get(LoadTy, AS)); - return ConstantFoldLoadFromConstPtr(Src, LoadTy, DL); + return ConstantFoldLoadFromValue(Src, LoadTy, DL); } /// This function is called when we have a diff --git a/llvm/test/Transforms/InstCombine/gep-alias-gep-load.ll b/llvm/test/Transforms/InstCombine/gep-alias-gep-load.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/gep-alias-gep-load.ll @@ -0,0 +1,11 @@ +; RUN: opt -passes=instcombine -S < %s | FileCheck %s + +; CHECK: ret i32 4 + +@a = internal alias i32, getelementptr ({[3 x i32]}, {[3 x i32]}* @b, i32 0, i32 0, i32 1) +@b = internal constant {[3 x i32]} {[3 x i32] [i32 2, i32 3, i32 4]} + +define i32 @f() { + %a = load i32, i32* getelementptr (i32, i32* @a, i64 1) + ret i32 %a +} diff --git a/llvm/test/Transforms/InstCombine/invariant.group-load.ll b/llvm/test/Transforms/InstCombine/invariant.group-load.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/invariant.group-load.ll @@ -0,0 +1,16 @@ +; RUN: opt -passes=instcombine -S < %s | FileCheck %s + +; CHECK: ret i64 3 + +@A = linkonce_odr hidden constant { i64, i64 } { i64 2, i64 3 } + +declare i8* @llvm.strip.invariant.group.p0i8(i8* %p) + +define i64 @f() { + %p = bitcast { i64, i64 }* @A to i8* + %a = call i8* @llvm.strip.invariant.group.p0i8(i8* %p) + %b = getelementptr i8, i8* %a, i32 8 + %c = bitcast i8* %b to i64* + %d = load i64, i64* %c + ret i64 %d +}