Index: lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -73,17 +73,7 @@ bool FoldOperands(unsigned, const R600InstrInfo *, std::vector &); bool FoldDotOperands(unsigned, const R600InstrInfo *, std::vector &); - static bool checkType(const Value *ptr, unsigned int addrspace); - - static bool isGlobalStore(const MemSDNode *N); - static bool isFlatStore(const MemSDNode *N); - static bool isLocalStore(const StoreSDNode *N); - bool isConstantLoad(const MemSDNode *N, int cbID) const; - bool isGlobalLoad(const MemSDNode *N) const; - bool isFlatLoad(const MemSDNode *N) const; - bool isLocalLoad(const LoadSDNode *N) const; - bool isUniformBr(const SDNode *N) const; SDNode *glueCopyToM0(SDNode *N) const; @@ -227,8 +217,7 @@ SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N) const { if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS || - !checkType(cast(N)->getMemOperand()->getValue(), - AMDGPUAS::LOCAL_ADDRESS)) + cast(N)->getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS) return N; const SITargetLowering& Lowering = @@ -492,55 +481,13 @@ SelectCode(N); } -bool AMDGPUDAGToDAGISel::checkType(const Value *Ptr, unsigned AS) { - assert(AS != 0 && "Use checkPrivateAddress instead."); - if (!Ptr) - return false; - - return Ptr->getType()->getPointerAddressSpace() == AS; -} - -bool AMDGPUDAGToDAGISel::isGlobalStore(const MemSDNode *N) { - if (!N->writeMem()) - return false; - return checkType(N->getMemOperand()->getValue(), AMDGPUAS::GLOBAL_ADDRESS); -} - -bool AMDGPUDAGToDAGISel::isLocalStore(const StoreSDNode *N) { - return checkType(N->getMemOperand()->getValue(), AMDGPUAS::LOCAL_ADDRESS); -} - -bool AMDGPUDAGToDAGISel::isFlatStore(const MemSDNode *N) { - if (!N->writeMem()) - return false; - return checkType(N->getMemOperand()->getValue(), AMDGPUAS::FLAT_ADDRESS); -} - bool AMDGPUDAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const { if (!N->readMem()) return false; - const Value *MemVal = N->getMemOperand()->getValue(); if (CbId == -1) - return checkType(MemVal, AMDGPUAS::CONSTANT_ADDRESS); - - return checkType(MemVal, AMDGPUAS::CONSTANT_BUFFER_0 + CbId); -} - -bool AMDGPUDAGToDAGISel::isGlobalLoad(const MemSDNode *N) const { - if (!N->readMem()) - return false; + return N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS; - return checkType(N->getMemOperand()->getValue(), AMDGPUAS::GLOBAL_ADDRESS); -} - -bool AMDGPUDAGToDAGISel::isLocalLoad(const LoadSDNode *N) const { - return checkType(N->getMemOperand()->getValue(), AMDGPUAS::LOCAL_ADDRESS); -} - -bool AMDGPUDAGToDAGISel::isFlatLoad(const MemSDNode *N) const { - if (!N->readMem()) - return false; - return checkType(N->getMemOperand()->getValue(), AMDGPUAS::FLAT_ADDRESS); + return N->getAddressSpace() == AMDGPUAS::CONSTANT_BUFFER_0 + CbId; } bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const { Index: lib/Target/AMDGPU/AMDGPUInstructions.td =================================================================== --- lib/Target/AMDGPU/AMDGPUInstructions.td +++ lib/Target/AMDGPU/AMDGPUInstructions.td @@ -187,26 +187,58 @@ def truncstorei16_private : PrivateStore ; def store_private : PrivateStore ; -def global_store : PatFrag<(ops node:$val, node:$ptr), - (store node:$val, node:$ptr), [{ - return isGlobalStore(dyn_cast(N)); -}]>; - -def global_store_atomic : PatFrag<(ops node:$val, node:$ptr), - (atomic_store node:$val, node:$ptr), [{ - return isGlobalStore(dyn_cast(N)); +class GlobalMemOp : PatFrag (N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; }]>; // Global address space loads -def global_load : PatFrag<(ops node:$ptr), (load node:$ptr), [{ - return isGlobalLoad(dyn_cast(N)); +class GlobalLoad : GlobalMemOp < + (ops node:$ptr), (op node:$ptr) +>; + +def global_load : GlobalLoad ; + +// Global address space stores +class GlobalStore : GlobalMemOp < + (ops node:$value, node:$ptr), (op node:$value, node:$ptr) +>; + +def global_store : GlobalStore ; +def global_store_atomic : GlobalStore; + + +class ConstantMemOp : PatFrag (N)->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS; }]>; // Constant address space loads -def constant_load : PatFrag<(ops node:$ptr), (load node:$ptr), [{ - return isConstantLoad(dyn_cast(N), -1); +class ConstantLoad : ConstantMemOp < + (ops node:$ptr), (op node:$ptr) +>; + +def constant_load : ConstantLoad; + +class LocalMemOp : PatFrag (N)->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; +}]>; + +// Local address space loads +class LocalLoad : LocalMemOp < + (ops node:$ptr), (op node:$ptr) +>; + +class LocalStore : LocalMemOp < + (ops node:$value, node:$ptr), (op node:$value, node:$ptr) +>; + +class FlatMemOp : PatFrag (N)->getAddressSPace() == AMDGPUAS::FLAT_ADDRESS; }]>; +class FlatLoad : FlatMemOp < + (ops node:$ptr), (op node:$ptr) +>; + class AZExtLoadBase : PatFrag<(ops node:$ptr), (ld_node node:$ptr), [{ LoadSDNode *L = cast(N); @@ -220,29 +252,14 @@ return cast(N)->getMemoryVT() == MVT::i8; }]>; -def az_extloadi8_global : PatFrag<(ops node:$ptr), (az_extloadi8 node:$ptr), [{ - return isGlobalLoad(dyn_cast(N)); -}]>; - -def sextloadi8_global : PatFrag<(ops node:$ptr), (sextloadi8 node:$ptr), [{ - return isGlobalLoad(dyn_cast(N)); -}]>; - -def az_extloadi8_constant : PatFrag<(ops node:$ptr), (az_extloadi8 node:$ptr), [{ - return isConstantLoad(dyn_cast(N), -1); -}]>; - -def sextloadi8_constant : PatFrag<(ops node:$ptr), (sextloadi8 node:$ptr), [{ - return isConstantLoad(dyn_cast(N), -1); -}]>; +def az_extloadi8_global : GlobalLoad ; +def sextloadi8_global : GlobalLoad ; -def az_extloadi8_local : PatFrag<(ops node:$ptr), (az_extloadi8 node:$ptr), [{ - return isLocalLoad(dyn_cast(N)); -}]>; +def az_extloadi8_constant : ConstantLoad ; +def sextloadi8_constant : ConstantLoad ; -def sextloadi8_local : PatFrag<(ops node:$ptr), (sextloadi8 node:$ptr), [{ - return isLocalLoad(dyn_cast(N)); -}]>; +def az_extloadi8_local : LocalLoad ; +def sextloadi8_local : LocalLoad ; def extloadi8_private : PrivateLoad ; def sextloadi8_private : PrivateLoad ; @@ -251,29 +268,14 @@ return cast(N)->getMemoryVT() == MVT::i16; }]>; -def az_extloadi16_global : PatFrag<(ops node:$ptr), (az_extloadi16 node:$ptr), [{ - return isGlobalLoad(dyn_cast(N)); -}]>; - -def sextloadi16_global : PatFrag<(ops node:$ptr), (sextloadi16 node:$ptr), [{ - return isGlobalLoad(dyn_cast(N)); -}]>; +def az_extloadi16_global : GlobalLoad ; +def sextloadi16_global : GlobalLoad ; -def az_extloadi16_constant : PatFrag<(ops node:$ptr), (az_extloadi16 node:$ptr), [{ - return isConstantLoad(dyn_cast(N), -1); -}]>; +def az_extloadi16_constant : ConstantLoad ; +def sextloadi16_constant : ConstantLoad ; -def sextloadi16_constant : PatFrag<(ops node:$ptr), (sextloadi16 node:$ptr), [{ - return isConstantLoad(dyn_cast(N), -1); -}]>; - -def az_extloadi16_local : PatFrag<(ops node:$ptr), (az_extloadi16 node:$ptr), [{ - return isLocalLoad(dyn_cast(N)); -}]>; - -def sextloadi16_local : PatFrag<(ops node:$ptr), (sextloadi16 node:$ptr), [{ - return isLocalLoad(dyn_cast(N)); -}]>; +def az_extloadi16_local : LocalLoad ; +def sextloadi16_local : LocalLoad ; def extloadi16_private : PrivateLoad ; def sextloadi16_private : PrivateLoad ; @@ -282,49 +284,20 @@ return cast(N)->getMemoryVT() == MVT::i32; }]>; -def az_extloadi32_global : PatFrag<(ops node:$ptr), - (az_extloadi32 node:$ptr), [{ - return isGlobalLoad(dyn_cast(N)); -}]>; - -def az_extloadi32_flat : PatFrag<(ops node:$ptr), - (az_extloadi32 node:$ptr), [{ - return isFlatLoad(dyn_cast(N)); -}]>; - -def az_extloadi32_constant : PatFrag<(ops node:$ptr), - (az_extloadi32 node:$ptr), [{ - return isConstantLoad(dyn_cast(N), -1); -}]>; - -def truncstorei8_global : PatFrag<(ops node:$val, node:$ptr), - (truncstorei8 node:$val, node:$ptr), [{ - return isGlobalStore(dyn_cast(N)); -}]>; +def az_extloadi32_global : GlobalLoad ; -def truncstorei16_global : PatFrag<(ops node:$val, node:$ptr), - (truncstorei16 node:$val, node:$ptr), [{ - return isGlobalStore(dyn_cast(N)); -}]>; +def az_extloadi32_flat : FlatLoad ; -def local_store : PatFrag<(ops node:$val, node:$ptr), - (store node:$val, node:$ptr), [{ - return isLocalStore(dyn_cast(N)); -}]>; +def az_extloadi32_constant : ConstantLoad ; -def truncstorei8_local : PatFrag<(ops node:$val, node:$ptr), - (truncstorei8 node:$val, node:$ptr), [{ - return isLocalStore(dyn_cast(N)); -}]>; +def truncstorei8_global : GlobalStore ; +def truncstorei16_global : GlobalStore ; -def truncstorei16_local : PatFrag<(ops node:$val, node:$ptr), - (truncstorei16 node:$val, node:$ptr), [{ - return isLocalStore(dyn_cast(N)); -}]>; +def local_store : LocalStore ; +def truncstorei8_local : LocalStore ; +def truncstorei16_local : LocalStore ; -def local_load : PatFrag<(ops node:$ptr), (load node:$ptr), [{ - return isLocalLoad(dyn_cast(N)); -}]>; +def local_load : LocalLoad ; class Aligned8Bytes : PatFrag (N)->getAlignment() % 8 == 0; Index: lib/Target/AMDGPU/SIInstrInfo.td =================================================================== --- lib/Target/AMDGPU/SIInstrInfo.td +++ lib/Target/AMDGPU/SIInstrInfo.td @@ -147,9 +147,10 @@ class flat_ld : PatFrag<(ops node:$ptr), (ld node:$ptr), [{ - return isFlatLoad(dyn_cast(N)) || - isGlobalLoad(dyn_cast(N)) || - isConstantLoad(cast(N), -1); + const MemSDNode *LD = cast(N); + return LD->getAddressSpace() == AMDGPUAS::FLAT_ADDRESS || + LD->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS || + LD->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS; }]>; def flat_load : flat_ld ; @@ -161,8 +162,9 @@ class flat_st : PatFrag<(ops node:$val, node:$ptr), (st node:$val, node:$ptr), [{ - return isFlatStore(dyn_cast(N)) || - isGlobalStore(dyn_cast(N)); + const MemSDNode *ST = cast(N); + return ST->getAddressSpace() == AMDGPUAS::FLAT_ADDRESS || + ST->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; }]>; def flat_store: flat_st ; @@ -187,7 +189,7 @@ def mubuf_load_atomic : MubufLoad ; def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ - return isConstantLoad(cast(N), -1) && + return cast(N)->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS && static_cast(getTargetLowering())->isMemOpUniform(N); }]>; @@ -211,7 +213,7 @@ >; def si_ld_local : PatFrag <(ops node:$ptr), (SIld_local node:$ptr), [{ - return isLocalLoad(cast(N)); + return cast(N)->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; }]>; def si_load_local : PatFrag <(ops node:$ptr), (si_ld_local node:$ptr), [{ @@ -248,7 +250,7 @@ def si_st_local : PatFrag < (ops node:$val, node:$ptr), (SIst_local node:$val, node:$ptr), [{ - return isLocalStore(cast(N)); + return cast(N)->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; }]>; def si_store_local : PatFrag <