Index: include/llvm/CodeGen/PseudoSourceValue.h =================================================================== --- include/llvm/CodeGen/PseudoSourceValue.h +++ include/llvm/CodeGen/PseudoSourceValue.h @@ -25,6 +25,7 @@ class MachineFrameInfo; class MachineMemOperand; class raw_ostream; +class TargetInstrInfo; raw_ostream &operator<<(raw_ostream &OS, const MachineMemOperand &MMO); class PseudoSourceValue; @@ -48,6 +49,7 @@ private: PSVKind Kind; + unsigned AddressSpace; friend raw_ostream &llvm::operator<<(raw_ostream &OS, const PseudoSourceValue* PSV); @@ -58,7 +60,7 @@ virtual void printCustom(raw_ostream &O) const; public: - explicit PseudoSourceValue(PSVKind Kind); + explicit PseudoSourceValue(PSVKind Kind, const TargetInstrInfo *TII); virtual ~PseudoSourceValue(); @@ -68,6 +70,9 @@ bool isGOT() const { return Kind == GOT; } bool isConstantPool() const { return Kind == ConstantPool; } bool isJumpTable() const { return Kind == JumpTable; } + + unsigned getAddressSpace() const { return AddressSpace; } + unsigned getTargetCustom() const { return (Kind >= TargetCustom) ? ((Kind+1) - TargetCustom) : 0; } @@ -91,8 +96,8 @@ const int FI; public: - explicit FixedStackPseudoSourceValue(int FI) - : PseudoSourceValue(FixedStack), FI(FI) {} + explicit FixedStackPseudoSourceValue(int FI, const TargetInstrInfo *TII) + : PseudoSourceValue(FixedStack, TII), FI(FI) {} static bool classof(const PseudoSourceValue *V) { return V->kind() == FixedStack; @@ -111,7 +116,7 @@ class CallEntryPseudoSourceValue : public PseudoSourceValue { protected: - CallEntryPseudoSourceValue(PSVKind Kind); + CallEntryPseudoSourceValue(PSVKind Kind, const TargetInstrInfo *TII); public: bool isConstant(const MachineFrameInfo *) const override; @@ -124,7 +129,7 @@ const GlobalValue *GV; public: - GlobalValuePseudoSourceValue(const GlobalValue *GV); + GlobalValuePseudoSourceValue(const GlobalValue *GV, const TargetInstrInfo *TII); static bool classof(const PseudoSourceValue *V) { return V->kind() == GlobalValueCallEntry; @@ -138,7 +143,7 @@ const char *ES; public: - ExternalSymbolPseudoSourceValue(const char *ES); + ExternalSymbolPseudoSourceValue(const char *ES, const TargetInstrInfo *TII); static bool classof(const PseudoSourceValue *V) { return V->kind() == ExternalSymbolCallEntry; @@ -149,6 +154,7 @@ /// Manages creation of pseudo source values. class PseudoSourceValueManager { + const TargetInstrInfo *TII; const PseudoSourceValue StackPSV, GOTPSV, JumpTablePSV, ConstantPoolPSV; std::map> FSValues; StringMap> @@ -158,7 +164,7 @@ GlobalCallEntries; public: - PseudoSourceValueManager(); + PseudoSourceValueManager(const TargetInstrInfo *TII); /// Return a pseudo source value referencing the area below the stack frame of /// a function, e.g., the argument space. Index: include/llvm/Target/TargetInstrInfo.h =================================================================== --- include/llvm/Target/TargetInstrInfo.h +++ include/llvm/Target/TargetInstrInfo.h @@ -25,6 +25,7 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/Support/BranchProbability.h" #include "llvm/Support/ErrorHandling.h" @@ -1029,6 +1030,13 @@ } public: + /// getAddressSpaceForPseudoSourceKind - Given the kind of memory + /// (e.g. stack) the target returns the corresponding address space. + virtual unsigned + getAddressSpaceForPseudoSourceKind(PseudoSourceValue::PSVKind Kind) const { + return 0; + } + /// unfoldMemoryOperand - Separate a single instruction which folded a load or /// a store or a load and a store into two or more instruction. If this is /// possible, returns true as well as the new instructions by reference. Index: lib/CodeGen/MachineFunction.cpp =================================================================== --- lib/CodeGen/MachineFunction.cpp +++ lib/CodeGen/MachineFunction.cpp @@ -147,7 +147,8 @@ "Can't create a MachineFunction using a Module with a " "Target-incompatible DataLayout attached\n"); - PSVManager = llvm::make_unique(); + PSVManager = + llvm::make_unique(getSubtarget().getInstrInfo()); } MachineFunction::~MachineFunction() { Index: lib/CodeGen/MachineInstr.cpp =================================================================== --- lib/CodeGen/MachineInstr.cpp +++ lib/CodeGen/MachineInstr.cpp @@ -563,7 +563,11 @@ /// getAddrSpace - Return the LLVM IR address space number that this pointer /// points into. unsigned MachinePointerInfo::getAddrSpace() const { - if (V.isNull() || V.is()) return 0; + if (V.isNull()) return 0; + + if (V.is()) + return V.get()->getAddressSpace(); + return cast(V.get()->getType())->getAddressSpace(); } Index: lib/CodeGen/PseudoSourceValue.cpp =================================================================== --- lib/CodeGen/PseudoSourceValue.cpp +++ lib/CodeGen/PseudoSourceValue.cpp @@ -14,6 +14,7 @@ #include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/ADT/STLExtras.h" #include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/Target/TargetInstrInfo.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/LLVMContext.h" #include "llvm/Support/ErrorHandling.h" @@ -24,7 +25,12 @@ "Stack", "GOT", "JumpTable", "ConstantPool", "FixedStack", "GlobalValueCallEntry", "ExternalSymbolCallEntry"}; -PseudoSourceValue::PseudoSourceValue(PSVKind Kind) : Kind(Kind) {} +PseudoSourceValue::PseudoSourceValue(PSVKind Kind, const TargetInstrInfo *TII) + : Kind(Kind) { + assert(TII != nullptr); + AddressSpace = TII->getAddressSpaceForPseudoSourceKind(Kind); +} + PseudoSourceValue::~PseudoSourceValue() {} @@ -75,8 +81,9 @@ OS << "FixedStack" << FI; } -CallEntryPseudoSourceValue::CallEntryPseudoSourceValue(PSVKind Kind) - : PseudoSourceValue(Kind) {} +CallEntryPseudoSourceValue::CallEntryPseudoSourceValue( + PSVKind Kind, const TargetInstrInfo *TII) + : PseudoSourceValue(Kind, TII) {} bool CallEntryPseudoSourceValue::isConstant(const MachineFrameInfo *) const { return false; @@ -91,16 +98,19 @@ } GlobalValuePseudoSourceValue::GlobalValuePseudoSourceValue( - const GlobalValue *GV) - : CallEntryPseudoSourceValue(GlobalValueCallEntry), GV(GV) {} - -ExternalSymbolPseudoSourceValue::ExternalSymbolPseudoSourceValue(const char *ES) - : CallEntryPseudoSourceValue(ExternalSymbolCallEntry), ES(ES) {} - -PseudoSourceValueManager::PseudoSourceValueManager() - : StackPSV(PseudoSourceValue::Stack), GOTPSV(PseudoSourceValue::GOT), - JumpTablePSV(PseudoSourceValue::JumpTable), - ConstantPoolPSV(PseudoSourceValue::ConstantPool) {} + const GlobalValue *GV, + const TargetInstrInfo *TII) + : CallEntryPseudoSourceValue(GlobalValueCallEntry, TII), GV(GV) {} +ExternalSymbolPseudoSourceValue::ExternalSymbolPseudoSourceValue( + const char *ES, const TargetInstrInfo *TII) + : CallEntryPseudoSourceValue(ExternalSymbolCallEntry, TII), ES(ES) {} + +PseudoSourceValueManager::PseudoSourceValueManager(const TargetInstrInfo *TIInfo) + : TII(TIInfo), + StackPSV(PseudoSourceValue::Stack, TII), + GOTPSV(PseudoSourceValue::GOT, TII), + JumpTablePSV(PseudoSourceValue::JumpTable, TII), + ConstantPoolPSV(PseudoSourceValue::ConstantPool, TII) {} const PseudoSourceValue *PseudoSourceValueManager::getStack() { return &StackPSV; @@ -116,10 +126,11 @@ return &JumpTablePSV; } -const PseudoSourceValue *PseudoSourceValueManager::getFixedStack(int FI) { +const PseudoSourceValue * +PseudoSourceValueManager::getFixedStack(int FI) { std::unique_ptr &V = FSValues[FI]; if (!V) - V = llvm::make_unique(FI); + V = llvm::make_unique(FI, TII); return V.get(); } @@ -128,7 +139,7 @@ std::unique_ptr &E = GlobalCallEntries[GV]; if (!E) - E = llvm::make_unique(GV); + E = llvm::make_unique(GV, TII); return E.get(); } @@ -137,6 +148,6 @@ std::unique_ptr &E = ExternalCallEntries[ES]; if (!E) - E = llvm::make_unique(ES); + E = llvm::make_unique(ES, TII); return E.get(); } Index: lib/Target/AMDGPU/SIInstrInfo.h =================================================================== --- lib/Target/AMDGPU/SIInstrInfo.h +++ lib/Target/AMDGPU/SIInstrInfo.h @@ -242,6 +242,8 @@ unsigned DstReg, ArrayRef Cond, unsigned TrueReg, unsigned FalseReg) const; + unsigned getAddressSpaceForPseudoSourceKind(PseudoSourceValue::PSVKind Kind) const; + bool areMemAccessesTriviallyDisjoint(MachineInstr &MIa, MachineInstr &MIb, AliasAnalysis *AA = nullptr) const override; Index: lib/Target/AMDGPU/SIInstrInfo.cpp =================================================================== --- lib/Target/AMDGPU/SIInstrInfo.cpp +++ lib/Target/AMDGPU/SIInstrInfo.cpp @@ -1760,6 +1760,25 @@ } } +unsigned SIInstrInfo::getAddressSpaceForPseudoSourceKind( + PseudoSourceValue::PSVKind Kind) const { + switch(Kind) { + case PseudoSourceValue::Stack: + case PseudoSourceValue::FixedStack: + return AMDGPUASI.PRIVATE_ADDRESS; + case PseudoSourceValue::ConstantPool: + case PseudoSourceValue::GOT: + case PseudoSourceValue::JumpTable: + case PseudoSourceValue::GlobalValueCallEntry: + case PseudoSourceValue::ExternalSymbolCallEntry: + case PseudoSourceValue::TargetCustom: + return AMDGPUASI.CONSTANT_ADDRESS; + default: + return AMDGPUASI.FLAT_ADDRESS; + } +} + + static void removeModOperands(MachineInstr &MI) { unsigned Opc = MI.getOpcode(); int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc, Index: lib/Target/AMDGPU/SIMachineFunctionInfo.h =================================================================== --- lib/Target/AMDGPU/SIMachineFunctionInfo.h +++ lib/Target/AMDGPU/SIMachineFunctionInfo.h @@ -18,6 +18,7 @@ #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/Target/TargetInstrInfo.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/Support/ErrorHandling.h" #include @@ -29,8 +30,8 @@ class AMDGPUImagePseudoSourceValue : public PseudoSourceValue { public: - explicit AMDGPUImagePseudoSourceValue() : - PseudoSourceValue(PseudoSourceValue::TargetCustom) { } + explicit AMDGPUImagePseudoSourceValue(const TargetInstrInfo *TII) : + PseudoSourceValue(PseudoSourceValue::TargetCustom, TII) { } bool isConstant(const MachineFrameInfo *) const override { // This should probably be true for most images, but we will start by being @@ -53,8 +54,8 @@ class AMDGPUBufferPseudoSourceValue : public PseudoSourceValue { public: - explicit AMDGPUBufferPseudoSourceValue() : - PseudoSourceValue(PseudoSourceValue::TargetCustom) { } + explicit AMDGPUBufferPseudoSourceValue(const TargetInstrInfo *TII) : + PseudoSourceValue(PseudoSourceValue::TargetCustom, TII) { } bool isConstant(const MachineFrameInfo *) const override { // This should probably be true for most images, but we will start by being Index: lib/Target/AMDGPU/SIMachineFunctionInfo.cpp =================================================================== --- lib/Target/AMDGPU/SIMachineFunctionInfo.cpp +++ lib/Target/AMDGPU/SIMachineFunctionInfo.cpp @@ -49,6 +49,8 @@ WavesPerEU(0, 0), DebuggerWorkGroupIDStackObjectIndices({{0, 0, 0}}), DebuggerWorkItemIDStackObjectIndices({{0, 0, 0}}), + BufferPSV(MF.getSubtarget().getInstrInfo()), + ImagePSV(MF.getSubtarget().getInstrInfo()), LDSWaveSpillSize(0), NumUserSGPRs(0), NumSystemSGPRs(0),