Index: include/polly/CodeGen/BlockGenerators.h =================================================================== --- include/polly/CodeGen/BlockGenerators.h +++ include/polly/CodeGen/BlockGenerators.h @@ -37,6 +37,7 @@ class MemoryAccess; class ScopArrayInfo; class IslExprBuilder; +class MemAccInst; /// @brief Generate a new basic block for a polyhedral statement. class BlockGenerator { @@ -483,9 +484,8 @@ /// @param NewAccesses A map from memory access ids to new ast expressions, /// which may contain new access expressions for certain /// memory accesses. - Value *generateLocationAccessed(ScopStmt &Stmt, const Instruction *Inst, - Value *Pointer, ValueMapT &BBMap, - LoopToScevMapT <S, + Value *generateLocationAccessed(ScopStmt &Stmt, MemAccInst Inst, + ValueMapT &BBMap, LoopToScevMapT <S, isl_id_to_ast_expr *NewAccesses); /// @param NewAccesses A map from memory access ids to new ast expressions, Index: include/polly/ScopInfo.h =================================================================== --- include/polly/ScopInfo.h +++ include/polly/ScopInfo.h @@ -65,6 +65,101 @@ class ScopStmt; class ScopInfo; +/// @brief Utility class to wrap the common members of LoadInst and StoreInst. +/// +/// This works like LLVM utility classes like CallSite, IntrinsicInst, +/// MemIntrinsic, MemTransferInst, etc. +/// Unfortunately it is not possible to have an implicit conversion from +/// LoadInst*/StoreInst* in C++. Such casts must be written explicitely using +/// llvm::cast. +class MemAccInst { +private: + Instruction *I; + +public: + MemAccInst() : I(nullptr) {} + MemAccInst(const MemAccInst &Inst) : I(Inst.I) {} + /* implicit */ MemAccInst(LoadInst &LI) : I(&LI) {} + /* implicit */ MemAccInst(LoadInst *LI) : I(LI) {} + /* implicit */ MemAccInst(StoreInst &SI) : I(&SI) {} + /* implicit */ MemAccInst(StoreInst *SI) : I(SI) {} + explicit MemAccInst(Instruction &I) : I(&I) { + assert(isa(I) || isa(I)); + } + explicit MemAccInst(Instruction *I) : I(I) { + assert(isa(I) || isa(I)); + } + + MemAccInst &operator=(const MemAccInst &Inst) { + I = Inst.I; + return *this; + } + MemAccInst &operator=(LoadInst &LI) { + I = &LI; + return *this; + } + MemAccInst &operator=(LoadInst *LI) { + I = LI; + return *this; + } + MemAccInst &operator=(StoreInst &SI) { + I = &SI; + return *this; + } + MemAccInst &operator=(StoreInst *SI) { + I = SI; + return *this; + } + + operator Instruction *() const { return asInstruction(); } + + Value *getOperand(unsigned i) const { return I->getOperand(i); } + BasicBlock *getParent() const { return I->getParent(); } + LLVMContext &getContext() const { return I->getContext(); } + + Value *getValueOperand() { + return isStore() ? asStore()->getValueOperand() : asLoad(); + } + const Value *getValueOperand() const { + return isStore() ? asStore()->getValueOperand() : asLoad(); + } + + Value *getPointerOperand() { + return getOperand(isStore() ? StoreInst::getPointerOperandIndex() + : LoadInst::getPointerOperandIndex()); + } + const Value *getPointerOperand() const { + return getOperand(isStore() ? StoreInst::getPointerOperandIndex() + : LoadInst::getPointerOperandIndex()); + } + + unsigned getAlignment() const { + return isStore() ? asStore()->getAlignment() : asLoad()->getAlignment(); + } + bool isVolatile() const { + return isStore() ? asStore()->isVolatile() : asLoad()->isVolatile(); + } + bool isSimple() const { + return isStore() ? asStore()->isSimple() : asLoad()->isSimple(); + } + AtomicOrdering getOrdering() const { + return isStore() ? asStore()->getOrdering() : asLoad()->getOrdering(); + } + bool isUnordered() const { + return isStore() ? asStore()->isUnordered() : asLoad()->isUnordered(); + } + + bool isLoad() const { return isa(I); } + LoadInst *asLoad() const { return cast(I); } + + bool isStore() const { return isa(I); } + StoreInst *asStore() const { return cast(I); } + + Instruction *asInstruction() const { return I; } +}; + +typedef DenseMap OutgoingValueMapTy; + //===---------------------------------------------------------------------===// /// @brief Enumeration of assumptions Polly can take. @@ -1894,7 +1989,7 @@ /// @param R The region on which to build the data access dictionary. /// @param BoxedLoops The set of loops that are overapproximated in @p R. /// @param ScopRIL The required invariant loads equivalence classes. - void buildMemoryAccess(Instruction *Inst, Loop *L, Region *R, + void buildMemoryAccess(MemAccInst Inst, Loop *L, Region *R, const ScopDetection::BoxedLoopsSetTy *BoxedLoops, const InvariantLoadsSetTy &ScopRIL); @@ -1977,7 +2072,7 @@ /// @param Sizes The array dimension's sizes. /// @param AccessValue Value read or written. /// @see ScopArrayInfo::MemoryKind - void addArrayAccess(Instruction *MemAccInst, MemoryAccess::AccessType Type, + void addArrayAccess(MemAccInst MemAccInst, MemoryAccess::AccessType Type, Value *BaseAddress, unsigned ElemBytes, bool IsAffine, ArrayRef Subscripts, ArrayRef Sizes, Value *AccessValue); Index: lib/Analysis/ScopInfo.cpp =================================================================== --- lib/Analysis/ScopInfo.cpp +++ lib/Analysis/ScopInfo.cpp @@ -2497,8 +2497,8 @@ continue; if (!MA->isRead()) HasWriteAccess.insert(MA->getBaseAddr()); - Instruction *Acc = MA->getAccessInstruction(); - PtrToAcc[getPointerOperand(*Acc)] = MA; + MemAccInst Acc(MA->getAccessInstruction()); + PtrToAcc[Acc.getPointerOperand()] = MA; AST.add(Acc); } } @@ -3703,28 +3703,16 @@ extern MapInsnToMemAcc InsnToMemAcc; void ScopInfo::buildMemoryAccess( - Instruction *Inst, Loop *L, Region *R, + MemAccInst Inst, Loop *L, Region *R, const ScopDetection::BoxedLoopsSetTy *BoxedLoops, const InvariantLoadsSetTy &ScopRIL) { - unsigned Size; - Type *SizeType; - Value *Val; - enum MemoryAccess::AccessType Type; - - if (LoadInst *Load = dyn_cast(Inst)) { - SizeType = Load->getType(); - Size = TD->getTypeAllocSize(SizeType); - Type = MemoryAccess::READ; - Val = Load; - } else { - StoreInst *Store = cast(Inst); - SizeType = Store->getValueOperand()->getType(); - Size = TD->getTypeAllocSize(SizeType); - Type = MemoryAccess::MUST_WRITE; - Val = Store->getValueOperand(); - } - auto Address = getPointerOperand(*Inst); + Value *Address = Inst.getPointerOperand(); + Value *Val = Inst.getValueOperand(); + Type *SizeType = Val->getType(); + unsigned Size = TD->getTypeAllocSize(SizeType); + enum MemoryAccess::AccessType Type = + Inst.isLoad() ? MemoryAccess::READ : MemoryAccess::MUST_WRITE; const SCEV *AccessFunction = SE->getSCEVAtScope(Address, L); const SCEVUnknown *BasePointer = @@ -3809,7 +3797,7 @@ // FIXME: Size of the number of bytes of an array element, not the number of // elements as probably intended here. const SCEV *SizeSCEV = - SE->getConstant(TD->getIntPtrType(Inst->getContext()), Size); + SE->getConstant(TD->getIntPtrType(Inst.getContext()), Size); if (!IsAffine && Type == MemoryAccess::MUST_WRITE) Type = MemoryAccess::MAY_WRITE; @@ -3882,7 +3870,7 @@ // there might be other invariant accesses that will be hoisted and // that would allow to make a non-affine access affine. if (isa(Inst) || isa(Inst)) - buildMemoryAccess(Inst, L, &R, BoxedLoops, ScopRIL); + buildMemoryAccess(MemAccInst(Inst), L, &R, BoxedLoops, ScopRIL); if (isIgnoredIntrinsic(Inst)) continue; @@ -3947,15 +3935,14 @@ return &AccList.back(); } -void ScopInfo::addArrayAccess(Instruction *MemAccInst, +void ScopInfo::addArrayAccess(MemAccInst MemAccInst, MemoryAccess::AccessType Type, Value *BaseAddress, unsigned ElemBytes, bool IsAffine, ArrayRef Subscripts, ArrayRef Sizes, Value *AccessValue) { - assert(isa(MemAccInst) || isa(MemAccInst)); - assert(isa(MemAccInst) == (Type == MemoryAccess::READ)); - addMemoryAccess(MemAccInst->getParent(), MemAccInst, Type, BaseAddress, + assert(MemAccInst.isLoad() == (Type == MemoryAccess::READ)); + addMemoryAccess(MemAccInst.getParent(), MemAccInst, Type, BaseAddress, ElemBytes, IsAffine, AccessValue, Subscripts, Sizes, ScopArrayInfo::MK_Array); } Index: lib/CodeGen/BlockGenerators.cpp =================================================================== --- lib/CodeGen/BlockGenerators.cpp +++ lib/CodeGen/BlockGenerators.cpp @@ -168,9 +168,10 @@ NewInst->setName("p_" + Inst->getName()); } -Value *BlockGenerator::generateLocationAccessed( - ScopStmt &Stmt, const Instruction *Inst, Value *Pointer, ValueMapT &BBMap, - LoopToScevMapT <S, isl_id_to_ast_expr *NewAccesses) { +Value * +BlockGenerator::generateLocationAccessed(ScopStmt &Stmt, MemAccInst Inst, + ValueMapT &BBMap, LoopToScevMapT <S, + isl_id_to_ast_expr *NewAccesses) { const MemoryAccess &MA = Stmt.getArrayAccessFor(Inst); isl_ast_expr *AccessExpr = isl_id_to_ast_expr_get(NewAccesses, MA.getId()); @@ -196,7 +197,8 @@ return Address; } - return getNewValue(Stmt, Pointer, BBMap, LTS, getLoopForInst(Inst)); + return getNewValue(Stmt, Inst.getPointerOperand(), BBMap, LTS, + getLoopForInst(Inst)); } Loop *BlockGenerator::getLoopForInst(const llvm::Instruction *Inst) { @@ -209,9 +211,8 @@ if (Value *PreloadLoad = GlobalMap.lookup(Load)) return PreloadLoad; - auto *Pointer = Load->getPointerOperand(); Value *NewPointer = - generateLocationAccessed(Stmt, Load, Pointer, BBMap, LTS, NewAccesses); + generateLocationAccessed(Stmt, Load, BBMap, LTS, NewAccesses); Value *ScalarLoad = Builder.CreateAlignedLoad( NewPointer, Load->getAlignment(), Load->getName() + "_p_scalar_"); @@ -225,9 +226,8 @@ void BlockGenerator::generateScalarStore(ScopStmt &Stmt, StoreInst *Store, ValueMapT &BBMap, LoopToScevMapT <S, isl_id_to_ast_expr *NewAccesses) { - auto *Pointer = Store->getPointerOperand(); Value *NewPointer = - generateLocationAccessed(Stmt, Store, Pointer, BBMap, LTS, NewAccesses); + generateLocationAccessed(Stmt, Store, BBMap, LTS, NewAccesses); Value *ValueOperand = getNewValue(Stmt, Store->getValueOperand(), BBMap, LTS, getLoopForInst(Store)); @@ -667,7 +667,7 @@ unsigned Offset = NegativeStride ? VectorWidth - 1 : 0; Value *NewPointer = nullptr; - NewPointer = generateLocationAccessed(Stmt, Load, Pointer, ScalarMaps[Offset], + NewPointer = generateLocationAccessed(Stmt, Load, ScalarMaps[Offset], VLTS[Offset], NewAccesses); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, "vector_ptr"); @@ -694,8 +694,8 @@ __isl_keep isl_id_to_ast_expr *NewAccesses) { auto *Pointer = Load->getPointerOperand(); Type *VectorPtrType = getVectorPtrTy(Pointer, 1); - Value *NewPointer = generateLocationAccessed(Stmt, Load, Pointer, BBMap, - VLTS[0], NewAccesses); + Value *NewPointer = + generateLocationAccessed(Stmt, Load, BBMap, VLTS[0], NewAccesses); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, Load->getName() + "_p_vec_p"); LoadInst *ScalarLoad = @@ -723,8 +723,8 @@ Value *Vector = UndefValue::get(VectorType); for (int i = 0; i < VectorWidth; i++) { - Value *NewPointer = generateLocationAccessed( - Stmt, Load, Pointer, ScalarMaps[i], VLTS[i], NewAccesses); + Value *NewPointer = generateLocationAccessed(Stmt, Load, ScalarMaps[i], + VLTS[i], NewAccesses); Value *ScalarLoad = Builder.CreateLoad(NewPointer, Load->getName() + "_p_scalar_"); Vector = Builder.CreateInsertElement( @@ -814,8 +814,8 @@ if (Access.isStrideOne(isl_map_copy(Schedule))) { Type *VectorPtrType = getVectorPtrTy(Pointer, getVectorWidth()); - Value *NewPointer = generateLocationAccessed( - Stmt, Store, Pointer, ScalarMaps[0], VLTS[0], NewAccesses); + Value *NewPointer = generateLocationAccessed(Stmt, Store, ScalarMaps[0], + VLTS[0], NewAccesses); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, "vector_ptr"); @@ -826,8 +826,8 @@ } else { for (unsigned i = 0; i < ScalarMaps.size(); i++) { Value *Scalar = Builder.CreateExtractElement(Vector, Builder.getInt32(i)); - Value *NewPointer = generateLocationAccessed( - Stmt, Store, Pointer, ScalarMaps[i], VLTS[i], NewAccesses); + Value *NewPointer = generateLocationAccessed(Stmt, Store, ScalarMaps[i], + VLTS[i], NewAccesses); Builder.CreateStore(Scalar, NewPointer); } }