Index: include/polly/CodeGen/BlockGenerators.h =================================================================== --- include/polly/CodeGen/BlockGenerators.h +++ include/polly/CodeGen/BlockGenerators.h @@ -67,40 +67,28 @@ /// The only public function exposed is generate(). class BlockGenerator { public: - /// @brief Generate a new BasicBlock for a ScopStmt. + BlockGenerator(PollyIRBuilder &Builder, Pass *P, LoopInfo &LI, + ScalarEvolution &SE, IslExprBuilder *ExprBuilder = nullptr); + + /// @brief Copy the basic block. + /// + /// This copies the entire basic block and updates references to old values + /// with references to new values, as defined by GlobalMap. /// - /// @param Builder The LLVM-IR Builder used to generate the statement. The - /// code is generated at the location, the Builder points to. /// @param Stmt The statement to code generate. - /// @param GlobalMap A map that defines for certain Values referenced from the - /// original code new Values they should be replaced with. - /// @param P A reference to the pass this function is called from. - /// The pass is needed to update other analysis. - /// @param LI The loop info for the current function - /// @param SE The scalar evolution info for the current function - /// @param Build The AST build with the new schedule. - /// @param ExprBuilder An expression builder to generate new access functions. - static void generate(PollyIRBuilder &Builder, ScopStmt &Stmt, - ValueMapT &GlobalMap, LoopToScevMapT <S, Pass *P, - LoopInfo &LI, ScalarEvolution &SE, - __isl_keep isl_ast_build *Build = nullptr, - IslExprBuilder *ExprBuilder = nullptr) { - BlockGenerator Generator(Builder, Stmt, P, LI, SE, Build, ExprBuilder); - Generator.copyBB(GlobalMap, LTS); - } + /// @param GlobalMap A mapping from old values to their new values + /// (for values recalculated in the new ScoP, but not + /// within this basic block). + /// @param LTS A map from old loops to new induction variables as SCEVs. + void copyBB(ScopStmt &Stmt, ValueMapT &GlobalMap, LoopToScevMapT <S); protected: PollyIRBuilder &Builder; - ScopStmt &Statement; Pass *P; LoopInfo &LI; ScalarEvolution &SE; - isl_ast_build *Build; IslExprBuilder *ExprBuilder; - - BlockGenerator(PollyIRBuilder &B, ScopStmt &Stmt, Pass *P, LoopInfo &LI, - ScalarEvolution &SE, __isl_keep isl_ast_build *Build, - IslExprBuilder *ExprBuilder); + BasicBlock &EntryBB; /// @brief Get the new version of a value. /// @@ -128,10 +116,10 @@ /// @returns o The old value, if it is still valid. /// o The new value, if available. /// o NULL, if no value is found. - Value *getNewValue(const Value *Old, ValueMapT &BBMap, ValueMapT &GlobalMap, - LoopToScevMapT <S, Loop *L) const; + Value *getNewValue(ScopStmt &Stmt, const Value *Old, ValueMapT &BBMap, + ValueMapT &GlobalMap, LoopToScevMapT <S, Loop *L) const; - void copyInstScalar(const Instruction *Inst, ValueMapT &BBMap, + void copyInstScalar(ScopStmt &Stmt, const Instruction *Inst, ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S); /// @brief Get the innermost loop that surrounds an instruction. @@ -141,18 +129,20 @@ Loop *getLoopForInst(const Instruction *Inst); /// @brief Get the new operand address according to access relation of @p MA. - Value *getNewAccessOperand(const MemoryAccess &MA); + Value *getNewAccessOperand(ScopStmt &Stmt, const MemoryAccess &MA); /// @brief Generate the operand address - Value *generateLocationAccessed(const Instruction *Inst, const Value *Pointer, - ValueMapT &BBMap, ValueMapT &GlobalMap, - LoopToScevMapT <S); + Value *generateLocationAccessed(ScopStmt &Stmt, const Instruction *Inst, + const Value *Pointer, ValueMapT &BBMap, + ValueMapT &GlobalMap, LoopToScevMapT <S); - Value *generateScalarLoad(const LoadInst *load, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S); + Value *generateScalarLoad(ScopStmt &Stmt, const LoadInst *load, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S); - Value *generateScalarStore(const StoreInst *store, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S); + Value *generateScalarStore(ScopStmt &Stmt, const StoreInst *store, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S); /// @brief Copy a single Instruction. /// @@ -164,8 +154,9 @@ /// @param GlobalMap A mapping from old values to their new values /// (for values recalculated in the new ScoP, but not /// within this basic block). - void copyInstruction(const Instruction *Inst, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S); + void copyInstruction(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S); /// @brief Copy the basic block. /// @@ -205,18 +196,13 @@ /// The pass is needed to update other analysis. /// @param LI The loop info for the current function /// @param SE The scalar evolution info for the current function - /// @param Build The AST build with the new schedule. /// @param ExprBuilder An expression builder to generate new access functions. - static void generate(PollyIRBuilder &B, ScopStmt &Stmt, + static void generate(BlockGenerator &BlockGen, ScopStmt &Stmt, VectorValueMapT &GlobalMaps, std::vector &VLTS, - __isl_keep isl_map *Schedule, Pass *P, LoopInfo &LI, - ScalarEvolution &SE, - __isl_keep isl_ast_build *Build = nullptr, - IslExprBuilder *ExprBuilder = nullptr) { - VectorBlockGenerator Generator(B, GlobalMaps, VLTS, Stmt, Schedule, P, LI, - SE, Build, ExprBuilder); - Generator.copyBB(); + __isl_keep isl_map *Schedule) { + VectorBlockGenerator Generator(BlockGen, GlobalMaps, VLTS, Schedule); + Generator.copyBB(Stmt); } private: @@ -249,16 +235,13 @@ // dimension of the innermost loop containing the statemenet. isl_map *Schedule; - VectorBlockGenerator(PollyIRBuilder &B, VectorValueMapT &GlobalMaps, - std::vector &VLTS, ScopStmt &Stmt, - __isl_keep isl_map *Schedule, Pass *P, LoopInfo &LI, - ScalarEvolution &SE, - __isl_keep isl_ast_build *Build = nullptr, - IslExprBuilder *ExprBuilder = nullptr); + VectorBlockGenerator(BlockGenerator &BlockGen, VectorValueMapT &GlobalMaps, + std::vector &VLTS, + __isl_keep isl_map *Schedule); int getVectorWidth(); - Value *getVectorValue(const Value *Old, ValueMapT &VectorMap, + Value *getVectorValue(ScopStmt &Stmt, const Value *Old, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps, Loop *L); Type *getVectorPtrTy(const Value *V, int Width); @@ -277,7 +260,7 @@ /// vector. By default we would do only positive /// strides. /// - Value *generateStrideOneLoad(const LoadInst *Load, + Value *generateStrideOneLoad(ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps, bool NegativeStride); @@ -291,7 +274,8 @@ /// %splat = shufflevector <1 x double> %splat_one, <1 x /// double> %splat_one, <4 x i32> zeroinitializer /// - Value *generateStrideZeroLoad(const LoadInst *Load, ValueMapT &BBMap); + Value *generateStrideZeroLoad(ScopStmt &Stmt, const LoadInst *Load, + ValueMapT &BBMap); /// @brief Load a vector from scalars distributed in memory /// @@ -304,33 +288,33 @@ /// %scalar 2 = load double* %p_2 /// %vec_2 = insertelement <2 x double> %vec_1, double %scalar_1, i32 1 /// - Value *generateUnknownStrideLoad(const LoadInst *Load, + Value *generateUnknownStrideLoad(ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps); - void generateLoad(const LoadInst *Load, ValueMapT &VectorMap, + void generateLoad(ScopStmt &Stmt, const LoadInst *Load, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyUnaryInst(const UnaryInstruction *Inst, ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps); + void copyUnaryInst(ScopStmt &Stmt, const UnaryInstruction *Inst, + ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyBinaryInst(const BinaryOperator *Inst, ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps); + void copyBinaryInst(ScopStmt &Stmt, const BinaryOperator *Inst, + ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyStore(const StoreInst *Store, ValueMapT &VectorMap, + void copyStore(ScopStmt &Stmt, const StoreInst *Store, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyInstScalarized(const Instruction *Inst, ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps); + void copyInstScalarized(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); bool extractScalarValues(const Instruction *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); bool hasVectorOperands(const Instruction *Inst, ValueMapT &VectorMap); - void copyInstruction(const Instruction *Inst, ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps); + void copyInstruction(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyBB(); + void copyBB(ScopStmt &Stmt); }; } #endif Index: include/polly/ScopInfo.h =================================================================== --- include/polly/ScopInfo.h +++ include/polly/ScopInfo.h @@ -46,6 +46,7 @@ struct isl_union_set; struct isl_union_map; struct isl_space; +struct isl_ast_build; struct isl_constraint; struct isl_pw_multi_aff; @@ -426,6 +427,9 @@ /// The BasicBlock represented by this statement. BasicBlock *BB; + /// @brief The isl AST build for the new generated AST. + isl_ast_build *Build; + std::vector NestLoops; std::string BaseName; @@ -559,6 +563,12 @@ const char *getBaseName() const; + /// @brief Set the isl AST build. + void setAstBuild(__isl_keep isl_ast_build *B) { Build = B; } + + /// @brief Get the isl AST build. + __isl_keep isl_ast_build *getAstBuild() const { return Build; } + /// @brief Restrict the domain of the statement. /// /// @param NewDomain The new statement domain. Index: lib/Analysis/ScopInfo.cpp =================================================================== --- lib/Analysis/ScopInfo.cpp +++ lib/Analysis/ScopInfo.cpp @@ -907,7 +907,7 @@ ScopStmt::ScopStmt(Scop &parent, TempScop &tempScop, const Region &CurRegion, BasicBlock &bb, SmallVectorImpl &Nest, SmallVectorImpl &Scatter) - : Parent(parent), BB(&bb), NestLoops(Nest.size()) { + : Parent(parent), BB(&bb), Build(nullptr), NestLoops(Nest.size()) { // Setup the induction variables. for (unsigned i = 0, e = Nest.size(); i < e; ++i) NestLoops[i] = Nest[i]; Index: lib/CodeGen/BlockGenerators.cpp =================================================================== --- lib/CodeGen/BlockGenerators.cpp +++ lib/CodeGen/BlockGenerators.cpp @@ -76,16 +76,14 @@ return false; } -BlockGenerator::BlockGenerator(PollyIRBuilder &B, ScopStmt &Stmt, Pass *P, - LoopInfo &LI, ScalarEvolution &SE, - isl_ast_build *Build, - IslExprBuilder *ExprBuilder) - : Builder(B), Statement(Stmt), P(P), LI(LI), SE(SE), Build(Build), - ExprBuilder(ExprBuilder) {} - -Value *BlockGenerator::getNewValue(const Value *Old, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S, - Loop *L) const { +BlockGenerator::BlockGenerator(PollyIRBuilder &B, Pass *P, LoopInfo &LI, + ScalarEvolution &SE, IslExprBuilder *ExprBuilder) + : Builder(B), P(P), LI(LI), SE(SE), ExprBuilder(ExprBuilder), + EntryBB(B.GetInsertBlock()->getParent()->getEntryBlock()) {} + +Value *BlockGenerator::getNewValue(ScopStmt &Stmt, const Value *Old, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S, Loop *L) const { // We assume constants never change. // This avoids map lookups for many calls to this function. if (isa(Old)) @@ -125,7 +123,7 @@ // A scop-constant value defined by an instruction executed outside the scop. if (const Instruction *Inst = dyn_cast(Old)) - if (!Statement.getParent()->getRegion().contains(Inst->getParent())) + if (!Stmt.getParent()->getRegion().contains(Inst->getParent())) return const_cast(Old); // The scalar dependence is neither available nor SCEVCodegenable. @@ -133,8 +131,9 @@ return nullptr; } -void BlockGenerator::copyInstScalar(const Instruction *Inst, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S) { +void BlockGenerator::copyInstScalar(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S) { // We do not generate debug intrinsics as we did not investigate how to // copy them correctly. At the current state, they just crash the code // generation as the meta-data operands are not correctly copied. @@ -145,8 +144,8 @@ // Replace old operands with the new ones. for (Value *OldOperand : Inst->operands()) { - Value *NewOperand = - getNewValue(OldOperand, BBMap, GlobalMap, LTS, getLoopForInst(Inst)); + Value *NewOperand = getNewValue(Stmt, OldOperand, BBMap, GlobalMap, LTS, + getLoopForInst(Inst)); if (!NewOperand) { assert(!isa(NewInst) && @@ -165,10 +164,12 @@ NewInst->setName("p_" + Inst->getName()); } -Value *BlockGenerator::getNewAccessOperand(const MemoryAccess &MA) { +Value *BlockGenerator::getNewAccessOperand(ScopStmt &Stmt, + const MemoryAccess &MA) { isl_pw_multi_aff *PWAccRel; isl_union_map *Schedule; isl_ast_expr *Expr; + isl_ast_build *Build = Stmt.getAstBuild(); assert(ExprBuilder && Build && "Cannot generate new value without IslExprBuilder!"); @@ -182,19 +183,17 @@ return ExprBuilder->create(Expr); } -Value *BlockGenerator::generateLocationAccessed(const Instruction *Inst, - const Value *Pointer, - ValueMapT &BBMap, - ValueMapT &GlobalMap, - LoopToScevMapT <S) { - const MemoryAccess &MA = Statement.getAccessFor(Inst); +Value *BlockGenerator::generateLocationAccessed( + ScopStmt &Stmt, const Instruction *Inst, const Value *Pointer, + ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S) { + const MemoryAccess &MA = Stmt.getAccessFor(Inst); Value *NewPointer; if (MA.hasNewAccessRelation()) - NewPointer = getNewAccessOperand(MA); + NewPointer = getNewAccessOperand(Stmt, MA); else NewPointer = - getNewValue(Pointer, BBMap, GlobalMap, LTS, getLoopForInst(Inst)); + getNewValue(Stmt, Pointer, BBMap, GlobalMap, LTS, getLoopForInst(Inst)); return NewPointer; } @@ -203,35 +202,36 @@ return LI.getLoopFor(Inst->getParent()); } -Value *BlockGenerator::generateScalarLoad(const LoadInst *Load, +Value *BlockGenerator::generateScalarLoad(ScopStmt &Stmt, const LoadInst *Load, ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S) { const Value *Pointer = Load->getPointerOperand(); Value *NewPointer = - generateLocationAccessed(Load, Pointer, BBMap, GlobalMap, LTS); + generateLocationAccessed(Stmt, Load, Pointer, BBMap, GlobalMap, LTS); Value *ScalarLoad = Builder.CreateAlignedLoad( NewPointer, Load->getAlignment(), Load->getName() + "_p_scalar_"); return ScalarLoad; } -Value *BlockGenerator::generateScalarStore(const StoreInst *Store, +Value *BlockGenerator::generateScalarStore(ScopStmt &Stmt, + const StoreInst *Store, ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S) { const Value *Pointer = Store->getPointerOperand(); Value *NewPointer = - generateLocationAccessed(Store, Pointer, BBMap, GlobalMap, LTS); - Value *ValueOperand = getNewValue(Store->getValueOperand(), BBMap, GlobalMap, - LTS, getLoopForInst(Store)); + generateLocationAccessed(Stmt, Store, Pointer, BBMap, GlobalMap, LTS); + Value *ValueOperand = getNewValue(Stmt, Store->getValueOperand(), BBMap, + GlobalMap, LTS, getLoopForInst(Store)); Value *NewStore = Builder.CreateAlignedStore(ValueOperand, NewPointer, Store->getAlignment()); return NewStore; } -void BlockGenerator::copyInstruction(const Instruction *Inst, ValueMapT &BBMap, - ValueMapT &GlobalMap, +void BlockGenerator::copyInstruction(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S) { // Terminator instructions control the control flow. They are explicitly // expressed in the clast and do not need to be copied. @@ -239,11 +239,11 @@ return; if (canSynthesize(Inst, &P->getAnalysis().getLoopInfo(), - &SE, &Statement.getParent()->getRegion())) + &SE, &Stmt.getParent()->getRegion())) return; if (const LoadInst *Load = dyn_cast(Inst)) { - Value *NewLoad = generateScalarLoad(Load, BBMap, GlobalMap, LTS); + Value *NewLoad = generateScalarLoad(Stmt, Load, BBMap, GlobalMap, LTS); // Compute NewLoad before its insertion in BBMap to make the insertion // deterministic. BBMap[Load] = NewLoad; @@ -251,7 +251,7 @@ } if (const StoreInst *Store = dyn_cast(Inst)) { - Value *NewStore = generateScalarStore(Store, BBMap, GlobalMap, LTS); + Value *NewStore = generateScalarStore(Stmt, Store, BBMap, GlobalMap, LTS); // Compute NewStore before its insertion in BBMap to make the insertion // deterministic. BBMap[Store] = NewStore; @@ -282,16 +282,17 @@ } } - copyInstScalar(Inst, BBMap, GlobalMap, LTS); + copyInstScalar(Stmt, Inst, BBMap, GlobalMap, LTS); } -void BlockGenerator::copyBB(ValueMapT &GlobalMap, LoopToScevMapT <S) { +void BlockGenerator::copyBB(ScopStmt &Stmt, ValueMapT &GlobalMap, + LoopToScevMapT <S) { auto *DTWP = P->getAnalysisIfAvailable(); auto *DT = DTWP ? &DTWP->getDomTree() : nullptr; auto *LIWP = P->getAnalysisIfAvailable(); auto *LI = LIWP ? &LIWP->getLoopInfo() : nullptr; - BasicBlock *BB = Statement.getBasicBlock(); + BasicBlock *BB = Stmt.getBasicBlock(); BasicBlock *CopyBB = SplitBlock(Builder.GetInsertBlock(), Builder.GetInsertPoint(), DT, LI); CopyBB->setName("polly.stmt." + BB->getName()); @@ -300,21 +301,20 @@ ValueMapT BBMap; for (Instruction &Inst : *BB) - copyInstruction(&Inst, BBMap, GlobalMap, LTS); + copyInstruction(Stmt, &Inst, BBMap, GlobalMap, LTS); } -VectorBlockGenerator::VectorBlockGenerator( - PollyIRBuilder &B, VectorValueMapT &GlobalMaps, - std::vector &VLTS, ScopStmt &Stmt, - __isl_keep isl_map *Schedule, Pass *P, LoopInfo &LI, ScalarEvolution &SE, - __isl_keep isl_ast_build *Build, IslExprBuilder *ExprBuilder) - : BlockGenerator(B, Stmt, P, LI, SE, Build, ExprBuilder), - GlobalMaps(GlobalMaps), VLTS(VLTS), Schedule(Schedule) { +VectorBlockGenerator::VectorBlockGenerator(BlockGenerator &BlockGen, + VectorValueMapT &GlobalMaps, + std::vector &VLTS, + isl_map *Schedule) + : BlockGenerator(BlockGen), GlobalMaps(GlobalMaps), VLTS(VLTS), + Schedule(Schedule) { assert(GlobalMaps.size() > 1 && "Only one vector lane found"); assert(Schedule && "No statement domain provided"); } -Value *VectorBlockGenerator::getVectorValue(const Value *Old, +Value *VectorBlockGenerator::getVectorValue(ScopStmt &Stmt, const Value *Old, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps, Loop *L) { @@ -327,8 +327,8 @@ for (int Lane = 0; Lane < Width; Lane++) Vector = Builder.CreateInsertElement( - Vector, - getNewValue(Old, ScalarMaps[Lane], GlobalMaps[Lane], VLTS[Lane], L), + Vector, getNewValue(Stmt, Old, ScalarMaps[Lane], GlobalMaps[Lane], + VLTS[Lane], L), Builder.getInt32(Lane)); VectorMap[Old] = Vector; @@ -346,17 +346,16 @@ return PointerType::getUnqual(VectorType); } -Value * -VectorBlockGenerator::generateStrideOneLoad(const LoadInst *Load, - VectorValueMapT &ScalarMaps, - bool NegativeStride = false) { +Value *VectorBlockGenerator::generateStrideOneLoad( + ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps, + bool NegativeStride = false) { unsigned VectorWidth = getVectorWidth(); const Value *Pointer = Load->getPointerOperand(); Type *VectorPtrType = getVectorPtrTy(Pointer, VectorWidth); unsigned Offset = NegativeStride ? VectorWidth - 1 : 0; Value *NewPointer = nullptr; - NewPointer = generateLocationAccessed(Load, Pointer, ScalarMaps[Offset], + NewPointer = generateLocationAccessed(Stmt, Load, Pointer, ScalarMaps[Offset], GlobalMaps[Offset], VLTS[Offset]); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, "vector_ptr"); @@ -378,12 +377,13 @@ return VecLoad; } -Value *VectorBlockGenerator::generateStrideZeroLoad(const LoadInst *Load, +Value *VectorBlockGenerator::generateStrideZeroLoad(ScopStmt &Stmt, + const LoadInst *Load, ValueMapT &BBMap) { const Value *Pointer = Load->getPointerOperand(); Type *VectorPtrType = getVectorPtrTy(Pointer, 1); - Value *NewPointer = - generateLocationAccessed(Load, Pointer, BBMap, GlobalMaps[0], VLTS[0]); + Value *NewPointer = generateLocationAccessed(Stmt, Load, Pointer, BBMap, + GlobalMaps[0], VLTS[0]); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, Load->getName() + "_p_vec_p"); LoadInst *ScalarLoad = @@ -400,9 +400,8 @@ return VectorLoad; } -Value * -VectorBlockGenerator::generateUnknownStrideLoad(const LoadInst *Load, - VectorValueMapT &ScalarMaps) { +Value *VectorBlockGenerator::generateUnknownStrideLoad( + ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps) { int VectorWidth = getVectorWidth(); const Value *Pointer = Load->getPointerOperand(); VectorType *VectorType = VectorType::get( @@ -411,8 +410,8 @@ Value *Vector = UndefValue::get(VectorType); for (int i = 0; i < VectorWidth; i++) { - Value *NewPointer = generateLocationAccessed(Load, Pointer, ScalarMaps[i], - GlobalMaps[i], VLTS[i]); + Value *NewPointer = generateLocationAccessed( + Stmt, Load, Pointer, ScalarMaps[i], GlobalMaps[i], VLTS[i]); Value *ScalarLoad = Builder.CreateLoad(NewPointer, Load->getName() + "_p_scalar_"); Vector = Builder.CreateInsertElement( @@ -422,18 +421,18 @@ return Vector; } -void VectorBlockGenerator::generateLoad(const LoadInst *Load, +void VectorBlockGenerator::generateLoad(ScopStmt &Stmt, const LoadInst *Load, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { if (PollyVectorizerChoice >= VECTORIZER_FIRST_NEED_GROUPED_UNROLL || !VectorType::isValidElementType(Load->getType())) { for (int i = 0; i < getVectorWidth(); i++) ScalarMaps[i][Load] = - generateScalarLoad(Load, ScalarMaps[i], GlobalMaps[i], VLTS[i]); + generateScalarLoad(Stmt, Load, ScalarMaps[i], GlobalMaps[i], VLTS[i]); return; } - const MemoryAccess &Access = Statement.getAccessFor(Load); + const MemoryAccess &Access = Stmt.getAccessFor(Load); // Make sure we have scalar values available to access the pointer to // the data location. @@ -441,23 +440,24 @@ Value *NewLoad; if (Access.isStrideZero(isl_map_copy(Schedule))) - NewLoad = generateStrideZeroLoad(Load, ScalarMaps[0]); + NewLoad = generateStrideZeroLoad(Stmt, Load, ScalarMaps[0]); else if (Access.isStrideOne(isl_map_copy(Schedule))) - NewLoad = generateStrideOneLoad(Load, ScalarMaps); + NewLoad = generateStrideOneLoad(Stmt, Load, ScalarMaps); else if (Access.isStrideX(isl_map_copy(Schedule), -1)) - NewLoad = generateStrideOneLoad(Load, ScalarMaps, true); + NewLoad = generateStrideOneLoad(Stmt, Load, ScalarMaps, true); else - NewLoad = generateUnknownStrideLoad(Load, ScalarMaps); + NewLoad = generateUnknownStrideLoad(Stmt, Load, ScalarMaps); VectorMap[Load] = NewLoad; } -void VectorBlockGenerator::copyUnaryInst(const UnaryInstruction *Inst, +void VectorBlockGenerator::copyUnaryInst(ScopStmt &Stmt, + const UnaryInstruction *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { int VectorWidth = getVectorWidth(); - Value *NewOperand = getVectorValue(Inst->getOperand(0), VectorMap, ScalarMaps, - getLoopForInst(Inst)); + Value *NewOperand = getVectorValue(Stmt, Inst->getOperand(0), VectorMap, + ScalarMaps, getLoopForInst(Inst)); assert(isa(Inst) && "Can not generate vector code for instruction"); @@ -466,7 +466,8 @@ VectorMap[Inst] = Builder.CreateCast(Cast->getOpcode(), NewOperand, DestType); } -void VectorBlockGenerator::copyBinaryInst(const BinaryOperator *Inst, +void VectorBlockGenerator::copyBinaryInst(ScopStmt &Stmt, + const BinaryOperator *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { Loop *L = getLoopForInst(Inst); @@ -474,21 +475,21 @@ Value *OpOne = Inst->getOperand(1); Value *NewOpZero, *NewOpOne; - NewOpZero = getVectorValue(OpZero, VectorMap, ScalarMaps, L); - NewOpOne = getVectorValue(OpOne, VectorMap, ScalarMaps, L); + NewOpZero = getVectorValue(Stmt, OpZero, VectorMap, ScalarMaps, L); + NewOpOne = getVectorValue(Stmt, OpOne, VectorMap, ScalarMaps, L); Value *NewInst = Builder.CreateBinOp(Inst->getOpcode(), NewOpZero, NewOpOne, Inst->getName() + "p_vec"); VectorMap[Inst] = NewInst; } -void VectorBlockGenerator::copyStore(const StoreInst *Store, +void VectorBlockGenerator::copyStore(ScopStmt &Stmt, const StoreInst *Store, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { - const MemoryAccess &Access = Statement.getAccessFor(Store); + const MemoryAccess &Access = Stmt.getAccessFor(Store); const Value *Pointer = Store->getPointerOperand(); - Value *Vector = getVectorValue(Store->getValueOperand(), VectorMap, + Value *Vector = getVectorValue(Stmt, Store->getValueOperand(), VectorMap, ScalarMaps, getLoopForInst(Store)); // Make sure we have scalar values available to access the pointer to @@ -497,8 +498,8 @@ if (Access.isStrideOne(isl_map_copy(Schedule))) { Type *VectorPtrType = getVectorPtrTy(Pointer, getVectorWidth()); - Value *NewPointer = generateLocationAccessed(Store, Pointer, ScalarMaps[0], - GlobalMaps[0], VLTS[0]); + Value *NewPointer = generateLocationAccessed( + Stmt, Store, Pointer, ScalarMaps[0], GlobalMaps[0], VLTS[0]); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, "vector_ptr"); @@ -510,7 +511,7 @@ for (unsigned i = 0; i < ScalarMaps.size(); i++) { Value *Scalar = Builder.CreateExtractElement(Vector, Builder.getInt32(i)); Value *NewPointer = generateLocationAccessed( - Store, Pointer, ScalarMaps[i], GlobalMaps[i], VLTS[i]); + Stmt, Store, Pointer, ScalarMaps[i], GlobalMaps[i], VLTS[i]); Builder.CreateStore(Scalar, NewPointer); } } @@ -556,7 +557,8 @@ return HasVectorOperand; } -void VectorBlockGenerator::copyInstScalarized(const Instruction *Inst, +void VectorBlockGenerator::copyInstScalarized(ScopStmt &Stmt, + const Instruction *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { bool HasVectorOperand; @@ -565,7 +567,7 @@ HasVectorOperand = extractScalarValues(Inst, VectorMap, ScalarMaps); for (int VectorLane = 0; VectorLane < getVectorWidth(); VectorLane++) - BlockGenerator::copyInstruction(Inst, ScalarMaps[VectorLane], + BlockGenerator::copyInstruction(Stmt, Inst, ScalarMaps[VectorLane], GlobalMaps[VectorLane], VLTS[VectorLane]); if (!VectorType::isValidElementType(Inst->getType()) || !HasVectorOperand) @@ -584,7 +586,8 @@ int VectorBlockGenerator::getVectorWidth() { return GlobalMaps.size(); } -void VectorBlockGenerator::copyInstruction(const Instruction *Inst, +void VectorBlockGenerator::copyInstruction(ScopStmt &Stmt, + const Instruction *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { // Terminator instructions control the control flow. They are explicitly @@ -593,27 +596,27 @@ return; if (canSynthesize(Inst, &P->getAnalysis().getLoopInfo(), - &SE, &Statement.getParent()->getRegion())) + &SE, &Stmt.getParent()->getRegion())) return; if (const LoadInst *Load = dyn_cast(Inst)) { - generateLoad(Load, VectorMap, ScalarMaps); + generateLoad(Stmt, Load, VectorMap, ScalarMaps); return; } if (hasVectorOperands(Inst, VectorMap)) { if (const StoreInst *Store = dyn_cast(Inst)) { - copyStore(Store, VectorMap, ScalarMaps); + copyStore(Stmt, Store, VectorMap, ScalarMaps); return; } if (const UnaryInstruction *Unary = dyn_cast(Inst)) { - copyUnaryInst(Unary, VectorMap, ScalarMaps); + copyUnaryInst(Stmt, Unary, VectorMap, ScalarMaps); return; } if (const BinaryOperator *Binary = dyn_cast(Inst)) { - copyBinaryInst(Binary, VectorMap, ScalarMaps); + copyBinaryInst(Stmt, Binary, VectorMap, ScalarMaps); return; } @@ -621,16 +624,16 @@ // generate vector code. } - copyInstScalarized(Inst, VectorMap, ScalarMaps); + copyInstScalarized(Stmt, Inst, VectorMap, ScalarMaps); } -void VectorBlockGenerator::copyBB() { +void VectorBlockGenerator::copyBB(ScopStmt &Stmt) { auto *DTWP = P->getAnalysisIfAvailable(); auto *DT = DTWP ? &DTWP->getDomTree() : nullptr; auto *LIWP = P->getAnalysisIfAvailable(); auto *LI = LIWP ? &LIWP->getLoopInfo() : nullptr; - BasicBlock *BB = Statement.getBasicBlock(); + BasicBlock *BB = Stmt.getBasicBlock(); BasicBlock *CopyBB = SplitBlock(Builder.GetInsertBlock(), Builder.GetInsertPoint(), DT, LI); CopyBB->setName("polly.stmt." + BB->getName()); @@ -654,5 +657,5 @@ ValueMapT VectorBlockMap; for (Instruction &Inst : *BB) - copyInstruction(&Inst, VectorBlockMap, ScalarBlockMap); + copyInstruction(Stmt, &Inst, VectorBlockMap, ScalarBlockMap); } Index: lib/CodeGen/IslCodeGeneration.cpp =================================================================== --- lib/CodeGen/IslCodeGeneration.cpp +++ lib/CodeGen/IslCodeGeneration.cpp @@ -64,7 +64,8 @@ DominatorTree &DT, Scop &S) : S(S), Builder(Builder), Annotator(Annotator), Rewriter(new SCEVExpander(SE, "polly")), - ExprBuilder(Builder, IDToValue, *Rewriter), P(P), DL(DL), LI(LI), + ExprBuilder(Builder, IDToValue, *Rewriter), + BlockGen(Builder, P, LI, SE, &ExprBuilder), P(P), DL(DL), LI(LI), SE(SE), DT(DT) {} ~IslNodeBuilder() { delete Rewriter; } @@ -82,6 +83,7 @@ SCEVExpander *Rewriter; IslExprBuilder ExprBuilder; + BlockGenerator BlockGen; Pass *P; const DataLayout &DL; LoopInfo &LI; @@ -398,6 +400,7 @@ isl_id *Id = isl_ast_expr_get_id(StmtExpr); isl_ast_expr_free(StmtExpr); ScopStmt *Stmt = (ScopStmt *)isl_id_get_user(Id); + Stmt->setAstBuild(IslAstInfo::getBuild(User)); VectorValueMapT VectorMap(IVS.size()); std::vector VLTS(IVS.size()); @@ -406,8 +409,7 @@ isl_map *S = isl_map_from_union_map(Schedule); createSubstitutionsVector(Expr, Stmt, VectorMap, VLTS, IVS, IteratorID); - VectorBlockGenerator::generate(Builder, *Stmt, VectorMap, VLTS, S, P, LI, SE, - IslAstInfo::getBuild(User), &ExprBuilder); + VectorBlockGenerator::generate(BlockGen, *Stmt, VectorMap, VLTS, S); isl_map_free(S); isl_id_free(Id); @@ -795,10 +797,10 @@ LTS.insert(OutsideLoopIterations.begin(), OutsideLoopIterations.end()); Stmt = (ScopStmt *)isl_id_get_user(Id); + Stmt->setAstBuild(IslAstInfo::getBuild(User)); createSubstitutions(Expr, Stmt, VMap, LTS); - BlockGenerator::generate(Builder, *Stmt, VMap, LTS, P, LI, SE, - IslAstInfo::getBuild(User), &ExprBuilder); + BlockGen.copyBB(*Stmt, VMap, LTS); isl_ast_node_free(User); isl_id_free(Id);