Index: llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -138,6 +138,16 @@ return createInstruction(Instruction::BinaryOps::Or, {LHS, RHS}); } + VPInstruction *createPhiInstruction(Instruction *Inst = nullptr) { + assert((Inst == nullptr || isa(Inst)) && + "PHINode or nullptr is expected during VPPHINode construction."); + VPInstruction *NewVPInst = new VPPHINode(); + NewVPInst->setUnderlyingValue(Inst); + if (BB) + BB->insert(NewVPInst, InsertPt); + return NewVPInst; + } + //===--------------------------------------------------------------------===// // RAII helpers. //===--------------------------------------------------------------------===// Index: llvm/lib/Transforms/Vectorize/VPlan.h =================================================================== --- llvm/lib/Transforms/Vectorize/VPlan.h +++ llvm/lib/Transforms/Vectorize/VPlan.h @@ -743,6 +743,98 @@ void print(raw_ostream &O, const Twine &Indent) const override; }; +/// Concrete class for PHI instruction. +class VPPHINode : public VPInstruction { +private: + SmallVector VPBBUsers; + +public: + using vpblock_iterator = SmallVectorImpl::iterator; + using const_vpblock_iterator = + SmallVectorImpl::const_iterator; + + explicit VPPHINode(void) : VPInstruction(Instruction::PHI, + ArrayRef()) {} + + vpblock_iterator block_begin(void) { return VPBBUsers.begin(); } + + vpblock_iterator block_end(void) { return VPBBUsers.end(); } + + const_vpblock_iterator block_begin(void) const { + return VPBBUsers.begin(); + } + + const_vpblock_iterator block_end(void) const { + return VPBBUsers.end(); + } + + iterator_range blocks(void) { + return make_range(block_begin(), block_end()); + } + + iterator_range blocks(void) const { + return make_range(block_begin(), block_end()); + } + + operand_range incoming_values(void) { return operands(); } + const_operand_range incoming_values(void) const { return operands(); } + + /// Return number of incoming values. + unsigned getNumIncomingValues(void) const { return getNumOperands(); } + + /// Return VPValue that corresponds to Idx'th incomming VPBasicBlock. + VPValue *getIncomingValue(const unsigned Idx) const { return getOperand(Idx); } + + /// Return VPValue that corresponds to incomming VPBB. + VPValue *getIncomingValue(const VPBasicBlock *VPBB) const { + auto Idx = getBlockIndexOrNone(VPBB); + assert(Idx && "Cannot find value for a given BB."); + return getIncomingValue(Idx.getValue()); + } + + /// Add an incoming value to the end of the PHI list + void addIncoming(VPValue *Value, VPBasicBlock *Block) { + assert(Value && "Value must not be null."); + assert(Block && "Block must not be null."); + addOperand(Value); + VPBBUsers.push_back(Block); + } + + /// Return incoming basic block number \p Idx. + VPBasicBlock *getIncomingBlock(const unsigned Idx) const { + return VPBBUsers[Idx]; + } + + /// Return index for a given \p Block. + int getBlockIndex(const VPBasicBlock *Block) const { + auto It = llvm::find(make_range(block_begin(), block_end()), Block); + if (It != block_end()) + return std::distance(block_begin(), It); + return -1; + } + + /// Return Optional index for a given basic block \p Block. + Optional getBlockIndexOrNone(const VPBasicBlock *Block) const { + int Idx = getBlockIndex(Block); + return Idx != -1 ? Optional(Idx) : None; + } + + /// Method to support type inquiry through isa, cast, and dyn_cast. + static inline bool classof(const VPInstruction *V) { + return V->getOpcode() == Instruction::PHI; + } + + /// Method to support type inquiry through isa, cast, and dyn_cast. + static inline bool classof(const VPValue *V) { + return isa(V) && classof(cast(V)); + } + + /// Method to support type inquiry through isa, cast, and dyn_cast. + static inline bool classof(const VPRecipeBase *V) { + return isa(V) && classof(cast(V)); + } +}; + /// A recipe for handling all phi nodes except for integer and FP inductions. class VPWidenPHIRecipe : public VPRecipeBase { private: Index: llvm/lib/Transforms/Vectorize/VPlan.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/VPlan.cpp +++ llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -347,10 +347,27 @@ O << Instruction::getOpcodeName(getOpcode()); } - for (const VPValue *Operand : operands()) { - O << " "; - Operand->printAsOperand(O); + if (auto *Phi = dyn_cast(this)) { + auto PrintValueWithBB = [&](const unsigned I) { + O << " "; + O << " [ "; + Phi->getIncomingValue(I)->printAsOperand(O); + O << ", "; + O << Phi->getIncomingBlock(I)->getName(); + O << " ]"; + }; + const unsigned Size = Phi->getNumIncomingValues(); + for (unsigned I = 0; I < Size - 1; ++I) { + PrintValueWithBB(I); + O << ","; + } + PrintValueWithBB(Size-1); } + else + for (const VPValue *Operand : operands()) { + O << " "; + Operand->printAsOperand(O); + } } /// Generate the code inside the body of the vectorized loop. Assumes a single Index: llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp +++ llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp @@ -92,13 +92,14 @@ for (auto *Phi : PhisToFix) { assert(IRDef2VPValue.count(Phi) && "Missing VPInstruction for PHINode."); VPValue *VPVal = IRDef2VPValue[Phi]; - assert(isa(VPVal) && "Expected VPInstruction for phi node."); - auto *VPPhi = cast(VPVal); + assert(isa(VPVal) && "Expected VPPHINode for phi node."); + auto *VPPhi = cast(VPVal); assert(VPPhi->getNumOperands() == 0 && - "Expected VPInstruction with no operands."); + "Expected VPPHINode with no operands."); - for (Value *Op : Phi->operands()) - VPPhi->addOperand(getOrCreateVPOperand(Op)); + for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) + VPPhi->addIncoming(getOrCreateVPOperand(Phi->getIncomingValue(I)), + getOrCreateVPBB(Phi->getIncomingBlock(I))); } } @@ -211,8 +212,7 @@ // Phi node's operands may have not been visited at this point. We create // an empty VPInstruction that we will fix once the whole plain CFG has // been built. - NewVPInst = cast(VPIRBuilder.createNaryOp( - Inst->getOpcode(), {} /*No operands*/, Inst)); + NewVPInst = cast(VPIRBuilder.createPhiInstruction(Inst)); PhisToFix.push_back(Phi); } else { // Translate LLVM-IR operands into VPValue operands and set them in the Index: llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp =================================================================== --- llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp +++ llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp @@ -41,7 +41,7 @@ auto Plan = buildHCFG(LoopHeader); VPBasicBlock *Entry = Plan->getEntry()->getEntryBasicBlock(); - EXPECT_NE(nullptr, Entry->getSingleSuccessor()); + EXPECT_TRUE(Entry->getSingleSuccessor()); EXPECT_EQ(0u, Entry->getNumPredecessors()); EXPECT_EQ(1u, Entry->getNumSuccessors()); EXPECT_EQ(nullptr, Entry->getCondBit()); @@ -52,8 +52,8 @@ EXPECT_EQ(2u, VecBB->getNumSuccessors()); auto Iter = VecBB->begin(); - VPInstruction *Phi = dyn_cast(&*Iter++); - EXPECT_EQ(Instruction::PHI, Phi->getOpcode()); + auto *Phi = dyn_cast(&*Iter++); + EXPECT_TRUE(Phi); VPInstruction *Idx = dyn_cast(&*Iter++); EXPECT_EQ(Instruction::GetElementPtr, Idx->getOpcode()); @@ -123,7 +123,7 @@ DeadInstructions); VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock(); - EXPECT_NE(nullptr, Entry->getSingleSuccessor()); + EXPECT_TRUE(Entry->getSingleSuccessor()); EXPECT_EQ(0u, Entry->getNumPredecessors()); EXPECT_EQ(1u, Entry->getNumSuccessors()); @@ -134,22 +134,22 @@ auto Iter = VecBB->begin(); auto *Phi = dyn_cast(&*Iter++); - EXPECT_NE(nullptr, Phi); + EXPECT_TRUE(Phi); auto *Idx = dyn_cast(&*Iter++); - EXPECT_NE(nullptr, Idx); + EXPECT_TRUE(Idx); auto *Load = dyn_cast(&*Iter++); - EXPECT_NE(nullptr, Load); + EXPECT_TRUE(Load); auto *Add = dyn_cast(&*Iter++); - EXPECT_NE(nullptr, Add); + EXPECT_TRUE(Add); auto *Store = dyn_cast(&*Iter++); - EXPECT_NE(nullptr, Store); + EXPECT_TRUE(Store); auto *LastWiden = dyn_cast(&*Iter++); - EXPECT_NE(nullptr, LastWiden); + EXPECT_TRUE(LastWiden); EXPECT_EQ(VecBB->end(), Iter); }