Index: include/llvm/CodeGen/PseudoSourceValue.h =================================================================== --- include/llvm/CodeGen/PseudoSourceValue.h +++ include/llvm/CodeGen/PseudoSourceValue.h @@ -25,6 +25,7 @@ class MachineFrameInfo; class MachineMemOperand; class raw_ostream; +class TargetInstrInfo; raw_ostream &operator<<(raw_ostream &OS, const MachineMemOperand &MMO); class PseudoSourceValue; @@ -48,6 +49,7 @@ private: PSVKind Kind; + unsigned AddressSpace; friend raw_ostream &llvm::operator<<(raw_ostream &OS, const PseudoSourceValue* PSV); @@ -58,7 +60,7 @@ virtual void printCustom(raw_ostream &O) const; public: - explicit PseudoSourceValue(PSVKind Kind); + explicit PseudoSourceValue(PSVKind Kind, const TargetInstrInfo *TII); virtual ~PseudoSourceValue(); @@ -68,6 +70,9 @@ bool isGOT() const { return Kind == GOT; } bool isConstantPool() const { return Kind == ConstantPool; } bool isJumpTable() const { return Kind == JumpTable; } + + unsigned getAddressSpace() const { return AddressSpace; } + unsigned getTargetCustom() const { return (Kind >= TargetCustom) ? ((Kind+1) - TargetCustom) : 0; } @@ -91,8 +96,8 @@ const int FI; public: - explicit FixedStackPseudoSourceValue(int FI) - : PseudoSourceValue(FixedStack), FI(FI) {} + explicit FixedStackPseudoSourceValue(int FI, const TargetInstrInfo *TII) + : PseudoSourceValue(FixedStack, TII), FI(FI) {} static bool classof(const PseudoSourceValue *V) { return V->kind() == FixedStack; @@ -111,7 +116,7 @@ class CallEntryPseudoSourceValue : public PseudoSourceValue { protected: - CallEntryPseudoSourceValue(PSVKind Kind); + CallEntryPseudoSourceValue(PSVKind Kind, const TargetInstrInfo *TII); public: bool isConstant(const MachineFrameInfo *) const override; @@ -124,7 +129,7 @@ const GlobalValue *GV; public: - GlobalValuePseudoSourceValue(const GlobalValue *GV); + GlobalValuePseudoSourceValue(const GlobalValue *GV, const TargetInstrInfo *TII); static bool classof(const PseudoSourceValue *V) { return V->kind() == GlobalValueCallEntry; @@ -138,7 +143,7 @@ const char *ES; public: - ExternalSymbolPseudoSourceValue(const char *ES); + ExternalSymbolPseudoSourceValue(const char *ES, const TargetInstrInfo *TII); static bool classof(const PseudoSourceValue *V) { return V->kind() == ExternalSymbolCallEntry; @@ -158,7 +163,7 @@ GlobalCallEntries; public: - PseudoSourceValueManager(); + PseudoSourceValueManager(const TargetInstrInfo *TII); /// Return a pseudo source value referencing the area below the stack frame of /// a function, e.g., the argument space. @@ -179,11 +184,14 @@ /// Return a pseudo source value referencing a fixed stack frame entry, /// e.g., a spill slot. - const PseudoSourceValue *getFixedStack(int FI); + const PseudoSourceValue *getFixedStack(int FI, const TargetInstrInfo *TII); - const PseudoSourceValue *getGlobalValueCallEntry(const GlobalValue *GV); + const PseudoSourceValue *getGlobalValueCallEntry(const GlobalValue *GV, + const TargetInstrInfo *TII); - const PseudoSourceValue *getExternalSymbolCallEntry(const char *ES); + const PseudoSourceValue *getExternalSymbolCallEntry( + const char *ES, + const TargetInstrInfo *TII); }; } // end namespace llvm Index: include/llvm/Target/TargetInstrInfo.h =================================================================== --- include/llvm/Target/TargetInstrInfo.h +++ include/llvm/Target/TargetInstrInfo.h @@ -25,6 +25,7 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/Support/BranchProbability.h" #include "llvm/Support/ErrorHandling.h" @@ -1029,6 +1030,13 @@ } public: + /// getAddressSpaceForPseudoSourceKind - Given the kind of memory + /// (e.g. stack) the target returns the corresponding address space. + virtual unsigned + getAddressSpaceForPseudoSourceKind(PseudoSourceValue::PSVKind Kind) const { + return 0; + } + /// unfoldMemoryOperand - Separate a single instruction which folded a load or /// a store or a load and a store into two or more instruction. If this is /// possible, returns true as well as the new instructions by reference. Index: lib/CodeGen/MIRParser/MIParser.cpp =================================================================== --- lib/CodeGen/MIRParser/MIParser.cpp +++ lib/CodeGen/MIRParser/MIParser.cpp @@ -2047,6 +2047,8 @@ } bool MIParser::parseMemoryPseudoSourceValue(const PseudoSourceValue *&PSV) { + const auto *TII = MF.getSubtarget().getInstrInfo(); + switch (Token.kind()) { case MIToken::kw_stack: PSV = MF.getPSVManager().getStack(); @@ -2064,7 +2066,7 @@ int FI; if (parseFixedStackFrameIndex(FI)) return true; - PSV = MF.getPSVManager().getFixedStack(FI); + PSV = MF.getPSVManager().getFixedStack(FI, TII); // The token was already consumed, so use return here instead of break. return false; } @@ -2072,7 +2074,7 @@ int FI; if (parseStackFrameIndex(FI)) return true; - PSV = MF.getPSVManager().getFixedStack(FI); + PSV = MF.getPSVManager().getFixedStack(FI, TII); // The token was already consumed, so use return here instead of break. return false; } @@ -2084,12 +2086,12 @@ GlobalValue *GV = nullptr; if (parseGlobalValue(GV)) return true; - PSV = MF.getPSVManager().getGlobalValueCallEntry(GV); + PSV = MF.getPSVManager().getGlobalValueCallEntry(GV, TII); break; } case MIToken::ExternalSymbol: PSV = MF.getPSVManager().getExternalSymbolCallEntry( - MF.createExternalSymbolName(Token.stringValue())); + MF.createExternalSymbolName(Token.stringValue()), TII); break; default: return error( Index: lib/CodeGen/MachineFunction.cpp =================================================================== --- lib/CodeGen/MachineFunction.cpp +++ lib/CodeGen/MachineFunction.cpp @@ -147,7 +147,8 @@ "Can't create a MachineFunction using a Module with a " "Target-incompatible DataLayout attached\n"); - PSVManager = llvm::make_unique(); + PSVManager = + llvm::make_unique(getSubtarget().getInstrInfo()); } MachineFunction::~MachineFunction() { Index: lib/CodeGen/MachineInstr.cpp =================================================================== --- lib/CodeGen/MachineInstr.cpp +++ lib/CodeGen/MachineInstr.cpp @@ -563,7 +563,11 @@ /// getAddrSpace - Return the LLVM IR address space number that this pointer /// points into. unsigned MachinePointerInfo::getAddrSpace() const { - if (V.isNull() || V.is()) return 0; + if (V.isNull()) return 0; + + if (V.is()) + return V.get()->getAddressSpace(); + return cast(V.get()->getType())->getAddressSpace(); } @@ -594,7 +598,8 @@ /// the specified FrameIndex. MachinePointerInfo MachinePointerInfo::getFixedStack(MachineFunction &MF, int FI, int64_t Offset) { - return MachinePointerInfo(MF.getPSVManager().getFixedStack(FI), Offset); + const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); + return MachinePointerInfo(MF.getPSVManager().getFixedStack(FI, TII), Offset); } MachinePointerInfo MachinePointerInfo::getJumpTable(MachineFunction &MF) { Index: lib/CodeGen/PseudoSourceValue.cpp =================================================================== --- lib/CodeGen/PseudoSourceValue.cpp +++ lib/CodeGen/PseudoSourceValue.cpp @@ -14,6 +14,7 @@ #include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/ADT/STLExtras.h" #include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/Target/TargetInstrInfo.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/LLVMContext.h" #include "llvm/Support/ErrorHandling.h" @@ -24,7 +25,15 @@ "Stack", "GOT", "JumpTable", "ConstantPool", "FixedStack", "GlobalValueCallEntry", "ExternalSymbolCallEntry"}; -PseudoSourceValue::PseudoSourceValue(PSVKind Kind) : Kind(Kind) {} +PseudoSourceValue::PseudoSourceValue(PSVKind Kind, const TargetInstrInfo *TII) + : Kind(Kind) { + if (TII == nullptr) { + AddressSpace = 0; + } else { + AddressSpace = TII->getAddressSpaceForPseudoSourceKind(Kind); + } +} + PseudoSourceValue::~PseudoSourceValue() {} @@ -75,8 +84,9 @@ OS << "FixedStack" << FI; } -CallEntryPseudoSourceValue::CallEntryPseudoSourceValue(PSVKind Kind) - : PseudoSourceValue(Kind) {} +CallEntryPseudoSourceValue::CallEntryPseudoSourceValue( + PSVKind Kind, const TargetInstrInfo *TII) + : PseudoSourceValue(Kind, TII) {} bool CallEntryPseudoSourceValue::isConstant(const MachineFrameInfo *) const { return false; @@ -91,16 +101,18 @@ } GlobalValuePseudoSourceValue::GlobalValuePseudoSourceValue( - const GlobalValue *GV) - : CallEntryPseudoSourceValue(GlobalValueCallEntry), GV(GV) {} - -ExternalSymbolPseudoSourceValue::ExternalSymbolPseudoSourceValue(const char *ES) - : CallEntryPseudoSourceValue(ExternalSymbolCallEntry), ES(ES) {} - -PseudoSourceValueManager::PseudoSourceValueManager() - : StackPSV(PseudoSourceValue::Stack), GOTPSV(PseudoSourceValue::GOT), - JumpTablePSV(PseudoSourceValue::JumpTable), - ConstantPoolPSV(PseudoSourceValue::ConstantPool) {} + const GlobalValue *GV, + const TargetInstrInfo *TII) + : CallEntryPseudoSourceValue(GlobalValueCallEntry, TII), GV(GV) {} +ExternalSymbolPseudoSourceValue::ExternalSymbolPseudoSourceValue( + const char *ES, const TargetInstrInfo *TII) + : CallEntryPseudoSourceValue(ExternalSymbolCallEntry, TII), ES(ES) {} + +PseudoSourceValueManager::PseudoSourceValueManager(const TargetInstrInfo *TII) + : StackPSV(PseudoSourceValue::Stack, TII), + GOTPSV(PseudoSourceValue::GOT, TII), + JumpTablePSV(PseudoSourceValue::JumpTable, TII), + ConstantPoolPSV(PseudoSourceValue::ConstantPool, TII) {} const PseudoSourceValue *PseudoSourceValueManager::getStack() { return &StackPSV; @@ -116,27 +128,30 @@ return &JumpTablePSV; } -const PseudoSourceValue *PseudoSourceValueManager::getFixedStack(int FI) { +const PseudoSourceValue * +PseudoSourceValueManager::getFixedStack(int FI, const TargetInstrInfo *TII) { std::unique_ptr &V = FSValues[FI]; if (!V) - V = llvm::make_unique(FI); + V = llvm::make_unique(FI, TII); return V.get(); } const PseudoSourceValue * -PseudoSourceValueManager::getGlobalValueCallEntry(const GlobalValue *GV) { +PseudoSourceValueManager::getGlobalValueCallEntry(const GlobalValue *GV, + const TargetInstrInfo *TII) { std::unique_ptr &E = GlobalCallEntries[GV]; if (!E) - E = llvm::make_unique(GV); + E = llvm::make_unique(GV, TII); return E.get(); } const PseudoSourceValue * -PseudoSourceValueManager::getExternalSymbolCallEntry(const char *ES) { +PseudoSourceValueManager::getExternalSymbolCallEntry( + const char *ES, const TargetInstrInfo *TII) { std::unique_ptr &E = ExternalCallEntries[ES]; if (!E) - E = llvm::make_unique(ES); + E = llvm::make_unique(ES, TII); return E.get(); } Index: lib/CodeGen/StackSlotColoring.cpp =================================================================== --- lib/CodeGen/StackSlotColoring.cpp +++ lib/CodeGen/StackSlotColoring.cpp @@ -319,7 +319,7 @@ if (NewFI == -1 || (NewFI == (int)SS)) continue; - const PseudoSourceValue *NewSV = MF.getPSVManager().getFixedStack(NewFI); + const PseudoSourceValue *NewSV = MF.getPSVManager().getFixedStack(NewFI, TII); SmallVectorImpl &RefMMOs = SSRefs[SS]; for (unsigned i = 0, e = RefMMOs.size(); i != e; ++i) RefMMOs[i]->setValue(NewSV); Index: lib/Target/AMDGPU/SIInstrInfo.h =================================================================== --- lib/Target/AMDGPU/SIInstrInfo.h +++ lib/Target/AMDGPU/SIInstrInfo.h @@ -242,6 +242,8 @@ unsigned DstReg, ArrayRef Cond, unsigned TrueReg, unsigned FalseReg) const; + unsigned getAddressSpaceForPseudoSourceKind(PseudoSourceValue::PSVKind Kind) const; + bool areMemAccessesTriviallyDisjoint(MachineInstr &MIa, MachineInstr &MIb, AliasAnalysis *AA = nullptr) const override; Index: lib/Target/AMDGPU/SIInstrInfo.cpp =================================================================== --- lib/Target/AMDGPU/SIInstrInfo.cpp +++ lib/Target/AMDGPU/SIInstrInfo.cpp @@ -1760,6 +1760,25 @@ } } +unsigned SIInstrInfo::getAddressSpaceForPseudoSourceKind( + PseudoSourceValue::PSVKind Kind) const { + switch(Kind) { + case PseudoSourceValue::Stack: + case PseudoSourceValue::FixedStack: + return AMDGPUASI.PRIVATE_ADDRESS; + case PseudoSourceValue::ConstantPool: + return AMDGPUASI.CONSTANT_ADDRESS; + case PseudoSourceValue::GOT: + case PseudoSourceValue::JumpTable: + case PseudoSourceValue::GlobalValueCallEntry: + case PseudoSourceValue::ExternalSymbolCallEntry: + case PseudoSourceValue::TargetCustom: + default: + return 0; + } +} + + static void removeModOperands(MachineInstr &MI) { unsigned Opc = MI.getOpcode(); int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc, Index: lib/Target/AMDGPU/SIMachineFunctionInfo.h =================================================================== --- lib/Target/AMDGPU/SIMachineFunctionInfo.h +++ lib/Target/AMDGPU/SIMachineFunctionInfo.h @@ -18,6 +18,7 @@ #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/Target/TargetInstrInfo.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/Support/ErrorHandling.h" #include @@ -29,8 +30,8 @@ class AMDGPUImagePseudoSourceValue : public PseudoSourceValue { public: - explicit AMDGPUImagePseudoSourceValue() : - PseudoSourceValue(PseudoSourceValue::TargetCustom) { } + explicit AMDGPUImagePseudoSourceValue(const TargetInstrInfo *TII) : + PseudoSourceValue(PseudoSourceValue::TargetCustom, TII) { } bool isConstant(const MachineFrameInfo *) const override { // This should probably be true for most images, but we will start by being @@ -53,8 +54,8 @@ class AMDGPUBufferPseudoSourceValue : public PseudoSourceValue { public: - explicit AMDGPUBufferPseudoSourceValue() : - PseudoSourceValue(PseudoSourceValue::TargetCustom) { } + explicit AMDGPUBufferPseudoSourceValue(const TargetInstrInfo *TII) : + PseudoSourceValue(PseudoSourceValue::TargetCustom, TII) { } bool isConstant(const MachineFrameInfo *) const override { // This should probably be true for most images, but we will start by being Index: lib/Target/AMDGPU/SIMachineFunctionInfo.cpp =================================================================== --- lib/Target/AMDGPU/SIMachineFunctionInfo.cpp +++ lib/Target/AMDGPU/SIMachineFunctionInfo.cpp @@ -49,6 +49,8 @@ WavesPerEU(0, 0), DebuggerWorkGroupIDStackObjectIndices({{0, 0, 0}}), DebuggerWorkItemIDStackObjectIndices({{0, 0, 0}}), + BufferPSV(MF.getSubtarget().getInstrInfo()), + ImagePSV(MF.getSubtarget().getInstrInfo()), LDSWaveSpillSize(0), NumUserSGPRs(0), NumSystemSGPRs(0), Index: lib/Target/Mips/MipsMachineFunction.cpp =================================================================== --- lib/Target/Mips/MipsMachineFunction.cpp +++ lib/Target/Mips/MipsMachineFunction.cpp @@ -87,11 +87,13 @@ return IsISR && (FI == ISRDataRegFI[0] || FI == ISRDataRegFI[1]); } MachinePointerInfo MipsFunctionInfo::callPtrInfo(const char *ES) { - return MachinePointerInfo(MF.getPSVManager().getExternalSymbolCallEntry(ES)); + return MachinePointerInfo( + MF.getPSVManager().getExternalSymbolCallEntry(ES, nullptr)); } MachinePointerInfo MipsFunctionInfo::callPtrInfo(const GlobalValue *GV) { - return MachinePointerInfo(MF.getPSVManager().getGlobalValueCallEntry(GV)); + return MachinePointerInfo( + MF.getPSVManager().getGlobalValueCallEntry(GV, nullptr)); } int MipsFunctionInfo::getMoveF64ViaSpillFI(const TargetRegisterClass *RC) {