diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -403,8 +403,10 @@ /// List of successor blocks. SmallVector Successors; - /// Successor selector, null for zero or single successor blocks. - VPValue *CondBit = nullptr; + /// Successor selector managed by a VPUser. For blocks with zero or one + /// successors, there is no operand. Otherwise there is exactly one operand + /// which is the branch condition. + VPUser CondBitUser; /// Current block predicate - null if the block does not need a predicate. VPValue *Predicate = nullptr; @@ -553,11 +555,29 @@ } /// \return the condition bit selecting the successor. - VPValue *getCondBit() { return CondBit; } + VPValue *getCondBit() { + if (CondBitUser.getNumOperands()) + return CondBitUser.getOperand(0); + return nullptr; + } - const VPValue *getCondBit() const { return CondBit; } + const VPValue *getCondBit() const { + if (CondBitUser.getNumOperands()) + return CondBitUser.getOperand(0); + return nullptr; + } - void setCondBit(VPValue *CV) { CondBit = CV; } + void setCondBit(VPValue *CV) { + if (!CV) { + if (CondBitUser.getNumOperands() == 1) + CondBitUser.removeLastOperand(); + return; + } + if (CondBitUser.getNumOperands() == 1) + CondBitUser.setOperand(0, CV); + else + CondBitUser.addOperand(CV); + } VPValue *getPredicate() { return Predicate; } @@ -581,7 +601,7 @@ VPValue *Condition) { assert(Successors.empty() && "Setting two successors when others exist."); assert(Condition && "Setting two successors without condition!"); - CondBit = Condition; + setCondBit(Condition); appendSuccessor(IfTrue); appendSuccessor(IfFalse); } @@ -601,7 +621,7 @@ /// Remove all the successors of this block and set to null its condition bit void clearSuccessors() { Successors.clear(); - CondBit = nullptr; + setCondBit(nullptr); } /// The method which generates the output IR that correspond to this @@ -1736,9 +1756,6 @@ /// Holds the VPLoopInfo analysis for this VPlan. VPLoopInfo VPLInfo; - /// Holds the condition bit values built during VPInstruction to VPRecipe transformation. - SmallVector VPCBVs; - public: VPlan(VPBlockBase *Entry = nullptr) : Entry(Entry) { if (Entry) @@ -1759,8 +1776,6 @@ delete BackedgeTakenCount; for (VPValue *Def : VPExternalDefs) delete Def; - for (VPValue *CBV : VPCBVs) - delete CBV; } /// Generate the IR code for this VPlan. @@ -1796,11 +1811,6 @@ VPExternalDefs.insert(VPVal); } - /// Add \p CBV to the vector of condition bit values. - void addCBV(VPValue *CBV) { - VPCBVs.push_back(CBV); - } - void addVPValue(Value *V) { assert(V && "Trying to add a null Value to VPlan"); assert(!Value2VPValue.count(V) && "Value already exists in VPlan"); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1132,9 +1132,6 @@ for (const VPValue *V : Plan.VPExternalDefs) assignSlot(V); - for (const VPValue *V : Plan.VPCBVs) - assignSlot(V); - if (Plan.BackedgeTakenCount) assignSlot(Plan.BackedgeTakenCount); diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp @@ -253,7 +253,13 @@ assert((PreheaderBB->getTerminator()->getNumSuccessors() == 1) && "Unexpected loop preheader"); VPBasicBlock *PreheaderVPBB = getOrCreateVPBB(PreheaderBB); - createVPInstructionsForVPBB(PreheaderVPBB, PreheaderBB); + for (auto &I : *PreheaderBB) { + if (I.getType()->isVoidTy()) + continue; + VPValue *VPV = new VPValue(&I); + Plan.addExternalDef(VPV); + IRDef2VPValue[&I] = VPV; + } // Create empty VPBB for Loop H so that we can link PH->H. VPBlockBase *HeaderVPBB = getOrCreateVPBB(TheLoop->getHeader()); // Preheader's predecessors will be set during the loop RPO traversal below. diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -24,17 +24,6 @@ auto *TopRegion = cast(Plan->getEntry()); ReversePostOrderTraversal RPOT(TopRegion->getEntry()); - // Condition bit VPValues get deleted during transformation to VPRecipes. - // Create new VPValues and save away as condition bits. These will be deleted - // after finalizing the vector IR basic blocks. - for (VPBlockBase *Base : RPOT) { - VPBasicBlock *VPBB = Base->getEntryBasicBlock(); - if (auto *CondBit = VPBB->getCondBit()) { - auto *NCondBit = new VPValue(CondBit->getUnderlyingValue()); - VPBB->setCondBit(NCondBit); - Plan->addCBV(NCondBit); - } - } for (VPBlockBase *Base : RPOT) { // Do not widen instructions in pre-header and exit blocks. if (Base->getNumPredecessors() == 0 || Base->getNumSuccessors() == 0) diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -229,6 +229,11 @@ New->addUser(*this); } + void removeLastOperand() { + VPValue *Op = Operands.pop_back_val(); + Op->removeUser(*this); + } + typedef SmallVectorImpl::iterator operand_iterator; typedef SmallVectorImpl::const_iterator const_operand_iterator; typedef iterator_range operand_range;