Index: lib/Transforms/Vectorize/CMakeLists.txt =================================================================== --- lib/Transforms/Vectorize/CMakeLists.txt +++ lib/Transforms/Vectorize/CMakeLists.txt @@ -1,6 +1,7 @@ add_llvm_library(LLVMVectorize LoadStoreVectorizer.cpp LoopVectorizationLegality.cpp + LoopVectorizationPlanner.cpp LoopVectorize.cpp SLPVectorizer.cpp Vectorize.cpp Index: lib/Transforms/Vectorize/LoopVectorizationPlanner.h =================================================================== --- lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -255,6 +255,14 @@ O << *Plan; } + /// Sinks instructions in Plan, depending on their underlying values in + /// SinkAfter. + // FIXME: Migrate to using a VPlan based mapping, once LoopVectorizationLegality::getSinkAfter + // is moved the VPlan. + static void + sinkInstructions(VPlanPtr &Plan, + DenseMap &SinkAfter); + protected: /// Collect the instructions from the original loop that would be trivially /// dead in the vectorized loop if generated. Index: lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp =================================================================== --- /dev/null +++ lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp @@ -0,0 +1,57 @@ +//===-- LoopVectorizationPlanner.cpp --------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file implements parts of the LoopVectorizationPlanner class. +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/PostOrderIterator.h" +#include "llvm/Support/Debug.h" +#include "LoopVectorizationPlanner.h" + +namespace llvm { + +#define DEBUG_TYPE "loop-vectorize" + +void LoopVectorizationPlanner::sinkInstructions( + VPlanPtr &Plan, DenseMap &SinkAfter) { + DenseMap SinkAfterInverse; + VPRegionBlock *TopRegion = dyn_cast(Plan->getEntry()); + ReversePostOrderTraversal RPOT(TopRegion->getEntry()); + for (VPBlockBase *Base : RPOT) { + VPBasicBlock *OriginalVPBB = Base->getEntryBasicBlock(); + for (VPRecipeBase &Ingredient : *OriginalVPBB) { + VPInstruction *VPInst = dyn_cast(&Ingredient); + if (!VPInst) + continue; + + Instruction *Instr = dyn_cast(VPInst->getUnderlyingValue()); + assert(Instr && "Need Instruction as underlying value"); + + // Move instructions to handle first-order recurrences, step 1: avoid + // handling this instruction until after we've handled the instruction it + // should follow. + auto SAIt = SinkAfter.find(Instr); + if (SAIt != SinkAfter.end()) { + DEBUG(dbgs() << "Sinking" << *SAIt->first << " after" << *SAIt->second + << " to vectorize a 1st order recurrence.\n"); + SinkAfterInverse[SAIt->second] = VPInst; + continue; + } + + // Move instructions to handle first-order recurrences, step 2: push the + // instruction to be sunk at its insertion point. + auto SAInvIt = SinkAfterInverse.find(Instr); + if (SAInvIt != SinkAfterInverse.end()) + SAInvIt->second->moveAfter(VPInst); + } + } +} + +} //end namespace llvm Index: lib/Transforms/Vectorize/VPlanValue.h =================================================================== --- lib/Transforms/Vectorize/VPlanValue.h +++ lib/Transforms/Vectorize/VPlanValue.h @@ -38,6 +38,8 @@ // and live-outs which the VPlan will need to fix accordingly. class VPValue { friend class VPBuilder; + friend class LoopVectorizationPlanner; + private: const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast). Index: unittests/Transforms/Vectorize/VPlanTest.cpp =================================================================== --- unittests/Transforms/Vectorize/VPlanTest.cpp +++ unittests/Transforms/Vectorize/VPlanTest.cpp @@ -8,6 +8,9 @@ //===----------------------------------------------------------------------===// #include "../lib/Transforms/Vectorize/VPlan.h" +#include "../lib/Transforms/Vectorize/LoopVectorizationPlanner.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" #include "gtest/gtest.h" namespace llvm { @@ -48,5 +51,46 @@ CHECK_ITERATOR(VPBB2, I4, I3, I5); } +TEST(VPInstructionTest, sinkInstructions) { + // Create some dummy instructions, we need those for the SinkAfter mapping. + LLVMContext Context; + Value *V = new Argument(Type::getInt32Ty(Context)); + Instruction *I1 = BinaryOperator::Create(Instruction::Add, V, V); + Instruction *I2 = BinaryOperator::Create(Instruction::Add, I1, V); + Instruction *I3 = BinaryOperator::Create(Instruction::Add, I2, V); + Instruction *I4 = BinaryOperator::Create(Instruction::Add, I3, V); + Instruction *I5 = BinaryOperator::Create(Instruction::Add, I4, V); + + VPBuilder Builder; + + VPBasicBlock *VPBB1 = new VPBasicBlock(); + Builder.setInsertPoint(VPBB1); + VPInstruction *VI1 = dyn_cast(Builder.createNaryOp(1, {}, I1)); + VPInstruction *VI2 = dyn_cast(Builder.createNaryOp(2, {}, I2)); + VPInstruction *VI3 = dyn_cast(Builder.createNaryOp(3, {}, I3)); + + VPBasicBlock *VPBB2 = new VPBasicBlock(); + Builder.setInsertPoint(VPBB2); + VPInstruction *VI4 = dyn_cast(Builder.createNaryOp(4, {}, I4)); + VPInstruction *VI5 = dyn_cast(Builder.createNaryOp(5, {}, I5)); + + VPBB1->setOneSuccessor(VPBB2); + + VPRegionBlock *TopRegion = + new VPRegionBlock("TopRegion", false /*isReplicator*/); + TopRegion->setEntry(VPBB1); + TopRegion->setExit(VPBB2); + auto Plan = llvm::make_unique(TopRegion); + + DenseMap SinkAfter; + SinkAfter[I1] = I3; + SinkAfter[I2] = I4; + + LoopVectorizationPlanner::sinkInstructions(Plan, SinkAfter); + + CHECK_ITERATOR((*VPBB1), VI3, VI1); + CHECK_ITERATOR((*VPBB2), VI4, VI2, VI5); +} + } // namespace } // namespace llvm