Index: llvm/include/llvm/CodeGen/MachineMemOperand.h =================================================================== --- llvm/include/llvm/CodeGen/MachineMemOperand.h +++ llvm/include/llvm/CodeGen/MachineMemOperand.h @@ -101,13 +101,15 @@ MOVolatile = 1u << 2, /// The memory access is non-temporal. MONonTemporal = 1u << 3, - /// The memory access is invariant. - MOInvariant = 1u << 4, + /// The memory access is dereferenceable (i.e., doesn't trap). + MODereferenceable = 1u << 4, + /// The memory access always returns the same value (or traps). + MOInvariant = 1u << 5, // Reserved for use by target-specific passes. - MOTargetFlag1 = 1u << 5, - MOTargetFlag2 = 1u << 6, - MOTargetFlag3 = 1u << 7, + MOTargetFlag1 = 1u << 6, + MOTargetFlag2 = 1u << 7, + MOTargetFlag3 = 1u << 8, LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ MOTargetFlag3) }; @@ -178,6 +180,7 @@ bool isStore() const { return FlagVals & MOStore; } bool isVolatile() const { return FlagVals & MOVolatile; } bool isNonTemporal() const { return FlagVals & MONonTemporal; } + bool isDereferenceable() const { return FlagVals & MODereferenceable; } bool isInvariant() const { return FlagVals & MOInvariant; } /// Returns true if this memory operation doesn't have any ordering Index: llvm/include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -422,11 +422,12 @@ uint16_t IsVolatile : 1; uint16_t IsNonTemporal : 1; + uint16_t IsDereferenceable : 1; uint16_t IsInvariant : 1; uint16_t SynchScope : 1; // enum SynchronizationScope uint16_t Ordering : 4; // enum AtomicOrdering }; - enum { NumMemSDNodeBits = NumSDNodeBits + 8 }; + enum { NumMemSDNodeBits = NumSDNodeBits + 9 }; class LSBaseSDNodeBitfields { friend class LSBaseSDNode; @@ -1102,10 +1103,9 @@ return Data; } - // We access subclass data here so that we can check consistency - // with MachineMemOperand information. bool isVolatile() const { return MemSDNodeBits.IsVolatile; } bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; } + bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; } bool isInvariant() const { return MemSDNodeBits.IsInvariant; } AtomicOrdering getOrdering() const { Index: llvm/lib/CodeGen/MIRParser/MILexer.h =================================================================== --- llvm/lib/CodeGen/MIRParser/MILexer.h +++ llvm/lib/CodeGen/MIRParser/MILexer.h @@ -54,6 +54,7 @@ kw_implicit_define, kw_def, kw_dead, + kw_dereferenceable, kw_killed, kw_undef, kw_internal, @@ -166,7 +167,7 @@ bool isMemoryOperandFlag() const { return Kind == kw_volatile || Kind == kw_non_temporal || - Kind == kw_invariant; + Kind == kw_dereferenceable || Kind == kw_invariant; } bool is(TokenKind K) const { return Kind == K; } Index: llvm/lib/CodeGen/MIRParser/MILexer.cpp =================================================================== --- llvm/lib/CodeGen/MIRParser/MILexer.cpp +++ llvm/lib/CodeGen/MIRParser/MILexer.cpp @@ -222,6 +222,7 @@ .Case("target-flags", MIToken::kw_target_flags) .Case("volatile", MIToken::kw_volatile) .Case("non-temporal", MIToken::kw_non_temporal) + .Case("dereferenceable", MIToken::kw_dereferenceable) .Case("invariant", MIToken::kw_invariant) .Case("align", MIToken::kw_align) .Case("stack", MIToken::kw_stack) Index: llvm/lib/CodeGen/MIRParser/MIParser.cpp =================================================================== --- llvm/lib/CodeGen/MIRParser/MIParser.cpp +++ llvm/lib/CodeGen/MIRParser/MIParser.cpp @@ -1766,6 +1766,9 @@ case MIToken::kw_non_temporal: Flags |= MachineMemOperand::MONonTemporal; break; + case MIToken::kw_dereferenceable: + Flags |= MachineMemOperand::MODereferenceable; + break; case MIToken::kw_invariant: Flags |= MachineMemOperand::MOInvariant; break; Index: llvm/lib/CodeGen/MIRPrinter.cpp =================================================================== --- llvm/lib/CodeGen/MIRPrinter.cpp +++ llvm/lib/CodeGen/MIRPrinter.cpp @@ -892,6 +892,8 @@ OS << "volatile "; if (Op.isNonTemporal()) OS << "non-temporal "; + if (Op.isDereferenceable()) + OS << "dereferenceable "; if (Op.isInvariant()) OS << "invariant "; if (Op.isLoad()) Index: llvm/lib/CodeGen/MachineInstr.cpp =================================================================== --- llvm/lib/CodeGen/MachineInstr.cpp +++ llvm/lib/CodeGen/MachineInstr.cpp @@ -650,10 +650,10 @@ OS << ")"; } - // Print nontemporal info. if (isNonTemporal()) OS << "(nontemporal)"; - + if (isDereferenceable()) + OS << "(dereferenceable)"; if (isInvariant()) OS << "(invariant)"; } @@ -1580,7 +1580,8 @@ for (MachineMemOperand *MMO : memoperands()) { if (MMO->isVolatile()) return false; if (MMO->isStore()) return false; - if (MMO->isInvariant()) continue; + if (MMO->isInvariant() && MMO->isDereferenceable()) + continue; // A load from a constant PseudoSourceValue is invariant. if (const PseudoSourceValue *PSV = MMO->getPseudoValue()) Index: llvm/lib/CodeGen/MachinePipeliner.cpp =================================================================== --- llvm/lib/CodeGen/MachinePipeliner.cpp +++ llvm/lib/CodeGen/MachinePipeliner.cpp @@ -3082,7 +3082,8 @@ MachineInstr::mmo_iterator NewMemRefs = MF.allocateMemRefsArray(NumRefs); unsigned Refs = 0; for (MachineMemOperand *MMO : NewMI.memoperands()) { - if (MMO->isVolatile() || MMO->isInvariant() || (!MMO->getValue())) { + if (MMO->isVolatile() || (MMO->isInvariant() && MMO->isDereferenceable()) || + (!MMO->getValue())) { NewMemRefs[Refs++] = MMO; continue; } Index: llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -14280,6 +14280,8 @@ MachineMemOperand::Flags MMOFlags = LLD->getMemOperand()->getFlags(); if (!RLD->isInvariant()) MMOFlags &= ~MachineMemOperand::MOInvariant; + if (!RLD->isDereferenceable()) + MMOFlags &= ~MachineMemOperand::MODereferenceable; if (LLD->getExtensionType() == ISD::NON_EXTLOAD) { // FIXME: Discards pointer and AA info. Load = DAG.getLoad(TheSelect->getValueType(0), SDLoc(TheSelect), Index: llvm/lib/CodeGen/SelectionDAG/FastISel.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/FastISel.cpp +++ llvm/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -2183,6 +2183,8 @@ bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal) != nullptr; bool IsInvariant = I->getMetadata(LLVMContext::MD_invariant_load) != nullptr; + bool IsDereferenceable = + I->getMetadata(LLVMContext::MD_dereferenceable) != nullptr; const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range); AAMDNodes AAInfo; @@ -2197,6 +2199,8 @@ Flags |= MachineMemOperand::MOVolatile; if (IsNonTemporal) Flags |= MachineMemOperand::MONonTemporal; + if (IsDereferenceable) + Flags |= MachineMemOperand::MODereferenceable; if (IsInvariant) Flags |= MachineMemOperand::MOInvariant; Index: llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -632,7 +632,8 @@ SDLoc dl(N); auto MMOFlags = - L->getMemOperand()->getFlags() & ~MachineMemOperand::MOInvariant; + L->getMemOperand()->getFlags() & + ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); SDValue NewL; if (L->getExtensionType() == ISD::NON_EXTLOAD) { NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl, @@ -2085,7 +2086,8 @@ // Load the value as an integer value with the same number of bits. EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); auto MMOFlags = - L->getMemOperand()->getFlags() & ~MachineMemOperand::MOInvariant; + L->getMemOperand()->getFlags() & + ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); SDValue newL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT, Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -5167,9 +5167,10 @@ ISD::MemIndexedMode AM) { LoadSDNode *LD = cast(OrigLoad); assert(LD->getOffset().isUndef() && "Load is already a indexed load!"); - // Don't propagate the invariant flag. + // Don't propagate the invariant or dereferenceable flags. auto MMOFlags = - LD->getMemOperand()->getFlags() & ~MachineMemOperand::MOInvariant; + LD->getMemOperand()->getFlags() & + ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, LD->getChain(), Base, Offset, LD->getPointerInfo(), LD->getMemoryVT(), LD->getAlignment(), MMOFlags, @@ -6716,6 +6717,7 @@ : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { MemSDNodeBits.IsVolatile = MMO->isVolatile(); MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal(); + MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable(); MemSDNodeBits.IsInvariant = MMO->isInvariant(); // We check here that the size of the memory operand fits within the size of Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -2010,7 +2010,8 @@ if (Global) { MachinePointerInfo MPInfo(Global); MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1); - auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant; + auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant | + MachineMemOperand::MODereferenceable; *MemRefs = MF.getMachineMemOperand(MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlignment(PtrTy)); Node->setMemRefs(MemRefs, MemRefs + 1); @@ -3473,17 +3474,8 @@ bool isVolatile = I.isVolatile(); bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr; - - // The IR notion of invariant_load only guarantees that all *non-faulting* - // invariant loads result in the same value. The MI notion of invariant load - // guarantees that the load can be legally moved to any location within its - // containing function. The MI notion of invariant_load is stronger than the - // IR notion of invariant_load -- an MI invariant_load is an IR invariant_load - // with a guarantee that the location being loaded from is dereferenceable - // throughout the function's lifetime. - - bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr && - isDereferenceablePointer(SV, DAG.getDataLayout()); + bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr; + bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout()); unsigned Alignment = I.getAlignment(); AAMDNodes AAInfo; @@ -3551,6 +3543,8 @@ MMOFlags |= MachineMemOperand::MONonTemporal; if (isInvariant) MMOFlags |= MachineMemOperand::MOInvariant; + if (isDereferenceable) + MMOFlags |= MachineMemOperand::MODereferenceable; SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A, MachinePointerInfo(SV, Offsets[i]), Alignment, Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -3445,11 +3445,12 @@ // The first entry in the descriptor is a function pointer that we must call // to obtain the address of the variable. SDValue Chain = DAG.getEntryNode(); - SDValue FuncTLVGet = - DAG.getLoad(MVT::i64, DL, Chain, DescAddr, - MachinePointerInfo::getGOT(DAG.getMachineFunction()), - /* Alignment = */ 8, MachineMemOperand::MONonTemporal | - MachineMemOperand::MOInvariant); + SDValue FuncTLVGet = DAG.getLoad( + MVT::i64, DL, Chain, DescAddr, + MachinePointerInfo::getGOT(DAG.getMachineFunction()), + /* Alignment = */ 8, + MachineMemOperand::MONonTemporal | MachineMemOperand::MOInvariant | + MachineMemOperand::MODereferenceable); Chain = FuncTLVGet.getValue(1); MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); Index: llvm/lib/Target/AMDGPU/R600ISelLowering.cpp =================================================================== --- llvm/lib/Target/AMDGPU/R600ISelLowering.cpp +++ llvm/lib/Target/AMDGPU/R600ISelLowering.cpp @@ -1564,8 +1564,9 @@ SDValue Arg = DAG.getLoad( ISD::UNINDEXED, Ext, VT, DL, Chain, DAG.getConstant(Offset, DL, MVT::i32), DAG.getUNDEF(MVT::i32), PtrInfo, - MemVT, /* Alignment = */ 4, - MachineMemOperand::MONonTemporal | MachineMemOperand::MOInvariant); + MemVT, /* Alignment = */ 4, MachineMemOperand::MONonTemporal | + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); // 4 is the preferred alignment for the CONSTANT memory space. InVals.push_back(Arg); Index: llvm/lib/Target/AMDGPU/SIISelLowering.cpp =================================================================== --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -581,8 +581,10 @@ SDValue Ptr = LowerParameterPtr(DAG, SL, Chain, Offset); return DAG.getLoad(ISD::UNINDEXED, ExtTy, VT, SL, Chain, Ptr, PtrOffset, - PtrInfo, MemVT, Align, MachineMemOperand::MONonTemporal | - MachineMemOperand::MOInvariant); + PtrInfo, MemVT, Align, + MachineMemOperand::MONonTemporal | + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); } SDValue SITargetLowering::LowerFormalArguments( @@ -1792,7 +1794,8 @@ MachinePointerInfo PtrInfo(V, StructOffset); return DAG.getLoad(MVT::i32, SL, QueuePtr.getValue(1), Ptr, PtrInfo, MinAlign(64, StructOffset), - MachineMemOperand::MOInvariant); + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); } SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op, @@ -1910,7 +1913,8 @@ MachinePointerInfo PtrInfo(UndefValue::get(PtrTy)); return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), GOTAddr, PtrInfo, Align, - MachineMemOperand::MOInvariant); + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); } SDValue SITargetLowering::lowerTRAP(SDValue Op, @@ -2130,9 +2134,10 @@ }; MachineMemOperand *MMO = MF.getMachineMemOperand( - MachinePointerInfo(), - MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, - VT.getStoreSize(), 4); + MachinePointerInfo(), + MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant, + VT.getStoreSize(), 4); return DAG.getMemIntrinsicNode(AMDGPUISD::LOAD_CONSTANT, DL, Op->getVTList(), Ops, VT, MMO); } Index: llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp =================================================================== --- llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp +++ llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp @@ -4155,7 +4155,9 @@ if (Subtarget.isGVIndirectSymbol(GV)) { MIB = BuildMI(MBB, MI, DL, get(LoadOpc), Reg); MIB.addReg(Reg, RegState::Kill).addImm(0); - auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant; + auto Flags = MachineMemOperand::MOLoad | + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant; MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand( MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 4, 4); MIB.addMemOperand(MMO); Index: llvm/lib/Target/ARM/ARMISelLowering.cpp =================================================================== --- llvm/lib/Target/ARM/ARMISelLowering.cpp +++ llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -2040,10 +2040,11 @@ Callee = DAG.getNode( ARMISD::WrapperPIC, dl, PtrVt, DAG.getTargetGlobalAddress(GV, dl, PtrVt, 0, ARMII::MO_NONLAZY)); - Callee = - DAG.getLoad(PtrVt, dl, DAG.getEntryNode(), Callee, - MachinePointerInfo::getGOT(DAG.getMachineFunction()), - /* Alignment = */ 0, MachineMemOperand::MOInvariant); + Callee = DAG.getLoad( + PtrVt, dl, DAG.getEntryNode(), Callee, + MachinePointerInfo::getGOT(DAG.getMachineFunction()), + /* Alignment = */ 0, MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); } else if (Subtarget->isTargetCOFF()) { assert(Subtarget->isTargetWindows() && "Windows is the only supported COFF target"); @@ -2755,11 +2756,12 @@ // The first entry in the descriptor is a function pointer that we must call // to obtain the address of the variable. SDValue Chain = DAG.getEntryNode(); - SDValue FuncTLVGet = - DAG.getLoad(MVT::i32, DL, Chain, DescAddr, - MachinePointerInfo::getGOT(DAG.getMachineFunction()), - /* Alignment = */ 4, MachineMemOperand::MONonTemporal | - MachineMemOperand::MOInvariant); + SDValue FuncTLVGet = DAG.getLoad( + MVT::i32, DL, Chain, DescAddr, + MachinePointerInfo::getGOT(DAG.getMachineFunction()), + /* Alignment = */ 4, + MachineMemOperand::MONonTemporal | MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); Chain = FuncTLVGet.getValue(1); MachineFunction &F = DAG.getMachineFunction(); Index: llvm/lib/Target/ARM/ARMInstrInfo.cpp =================================================================== --- llvm/lib/Target/ARM/ARMInstrInfo.cpp +++ llvm/lib/Target/ARM/ARMInstrInfo.cpp @@ -123,7 +123,9 @@ MIB = BuildMI(MBB, MI, DL, get(ARM::MOV_ga_pcrel_ldr), Reg) .addGlobalAddress(GV, 0, ARMII::MO_NONLAZY); - auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant; + auto Flags = MachineMemOperand::MOLoad | + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant; MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand( MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 4, 4); MIB.addMemOperand(MMO); Index: llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp =================================================================== --- llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp +++ llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -2238,7 +2238,8 @@ SDValue P = DAG.getLoad( EltVT, dl, Root, Arg, MachinePointerInfo(SrcValue), DL.getABITypeAlignment(EltVT.getTypeForEVT(F->getContext())), - MachineMemOperand::MOInvariant); + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); if (P.getNode()) P.getNode()->setIROrder(idx + 1); @@ -2255,7 +2256,8 @@ SDValue P = DAG.getLoad( VecVT, dl, Root, Arg, MachinePointerInfo(SrcValue), DL.getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())), - MachineMemOperand::MOInvariant); + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); if (P.getNode()) P.getNode()->setIROrder(idx + 1); @@ -2297,7 +2299,8 @@ SDValue P = DAG.getLoad( VecVT, dl, Root, SrcAddr, MachinePointerInfo(SrcValue), DL.getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())), - MachineMemOperand::MOInvariant); + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); if (P.getNode()) P.getNode()->setIROrder(idx + 1); Index: llvm/lib/Target/PowerPC/PPCISelLowering.h =================================================================== --- llvm/lib/Target/PowerPC/PPCISelLowering.h +++ llvm/lib/Target/PowerPC/PPCISelLowering.h @@ -761,12 +761,24 @@ SDValue Chain; SDValue ResChain; MachinePointerInfo MPI; + bool IsDereferenceable; bool IsInvariant; unsigned Alignment; AAMDNodes AAInfo; const MDNode *Ranges; - ReuseLoadInfo() : IsInvariant(false), Alignment(0), Ranges(nullptr) {} + ReuseLoadInfo() + : IsDereferenceable(false), IsInvariant(false), Alignment(0), + Ranges(nullptr) {} + + MachineMemOperand::Flags MMOFlags() const { + MachineMemOperand::Flags F = MachineMemOperand::MONone; + if (IsDereferenceable) + F |= MachineMemOperand::MODereferenceable; + if (IsInvariant) + F |= MachineMemOperand::MOInvariant; + return F; + } }; bool canReuseLoadAddress(SDValue Op, EVT MemVT, ReuseLoadInfo &RLI, Index: llvm/lib/Target/PowerPC/PPCISelLowering.cpp =================================================================== --- llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -4444,7 +4444,8 @@ LDChain = CallSeqStart.getValue(CallSeqStart->getNumValues()-2); auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors() - ? MachineMemOperand::MOInvariant + ? (MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant) : MachineMemOperand::MONone; MachinePointerInfo MPI(CS ? CS->getCalledValue() : nullptr); @@ -6481,10 +6482,7 @@ LowerFP_TO_INTForReuse(Op, RLI, DAG, dl); return DAG.getLoad(Op.getValueType(), dl, RLI.Chain, RLI.Ptr, RLI.MPI, - RLI.Alignment, - RLI.IsInvariant ? MachineMemOperand::MOInvariant - : MachineMemOperand::MONone, - RLI.AAInfo, RLI.Ranges); + RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges); } // We're trying to insert a regular store, S, and then a load, L. If the @@ -6527,6 +6525,7 @@ RLI.Chain = LD->getChain(); RLI.MPI = LD->getPointerInfo(); + RLI.IsDereferenceable = LD->isDereferenceable(); RLI.IsInvariant = LD->isInvariant(); RLI.Alignment = LD->getAlignment(); RLI.AAInfo = LD->getAAInfo(); @@ -6719,11 +6718,8 @@ MachineFunction &MF = DAG.getMachineFunction(); if (canReuseLoadAddress(SINT, MVT::i64, RLI, DAG)) { - Bits = - DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI, RLI.Alignment, - RLI.IsInvariant ? MachineMemOperand::MOInvariant - : MachineMemOperand::MONone, - RLI.AAInfo, RLI.Ranges); + Bits = DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI, + RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges); spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); } else if (Subtarget.hasLFIWAX() && canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) { Index: llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp =================================================================== --- llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp +++ llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp @@ -1176,7 +1176,7 @@ return false; // There's no chance of overlap if the load is invariant. - if (Load->isInvariant()) + if (Load->isInvariant() && Load->isDereferenceable()) return true; // Otherwise we need to check whether there's an alias. Index: llvm/lib/Target/X86/X86InstrInfo.cpp =================================================================== --- llvm/lib/Target/X86/X86InstrInfo.cpp +++ llvm/lib/Target/X86/X86InstrInfo.cpp @@ -5684,7 +5684,9 @@ unsigned Reg = MIB->getOperand(0).getReg(); const GlobalValue *GV = cast((*MIB->memoperands_begin())->getValue()); - auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant; + auto Flags = MachineMemOperand::MOLoad | + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant; MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand( MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 8, 8); MachineBasicBlock::iterator I = MIB.getInstr();