diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -1539,6 +1539,11 @@ /// @} + /// \returns True if the target has determined this read can be sunk + /// across critical edges. + bool canAlwaysSinkRead(const Instruction *Inst, + const BasicBlock *Dest) const; + private: /// The abstract base class used to type erase specific TTI /// implementations. @@ -1887,6 +1892,9 @@ Align Alignment) const = 0; virtual VPLegalization getVPLegalizationStrategy(const VPIntrinsic &PI) const = 0; + + virtual bool canAlwaysSinkRead(const Instruction *Inst, + const BasicBlock *Dest) const = 0; }; template @@ -2550,6 +2558,11 @@ getVPLegalizationStrategy(const VPIntrinsic &PI) const override { return Impl.getVPLegalizationStrategy(PI); } + + bool canAlwaysSinkRead(const Instruction *Inst, + const BasicBlock *Dest) const override { + return Impl.canAlwaysSinkRead(Inst, Dest); + } }; template diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -848,6 +848,11 @@ /* OperatorStrategy */ TargetTransformInfo::VPLegalization::Convert); } + bool canAlwaysSinkRead(const Instruction *Inst, + const BasicBlock *Dest) const { + return false; + } + protected: // Obtain the minimum required size to hold the value (without the sign) // In case of a vector it returns the min required size for one element. diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -1171,6 +1171,11 @@ return TTIImpl->hasActiveVectorLength(Opcode, DataType, Alignment); } +bool TargetTransformInfo::canAlwaysSinkRead(const Instruction *Inst, + const BasicBlock *Dest) const { + return TTIImpl->canAlwaysSinkRead(Inst, Dest); +} + TargetTransformInfo::Concept::~Concept() = default; TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {} diff --git a/llvm/lib/Transforms/Scalar/Sink.cpp b/llvm/lib/Transforms/Scalar/Sink.cpp --- a/llvm/lib/Transforms/Scalar/Sink.cpp +++ b/llvm/lib/Transforms/Scalar/Sink.cpp @@ -15,6 +15,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/InitializePasses.h" #include "llvm/Support/Debug.h" @@ -63,7 +64,8 @@ /// IsAcceptableTarget - Return true if it is possible to sink the instruction /// in the specified basic block. static bool IsAcceptableTarget(Instruction *Inst, BasicBlock *SuccToSinkTo, - DominatorTree &DT, LoopInfo &LI) { + DominatorTree &DT, LoopInfo &LI, + TargetTransformInfo &TTI) { assert(Inst && "Instruction to be sunk is null"); assert(SuccToSinkTo && "Candidate sink target is null"); @@ -79,7 +81,7 @@ if (SuccToSinkTo->getUniquePredecessor() != Inst->getParent()) { // We cannot sink a load across a critical edge - there may be stores in // other code paths. - if (Inst->mayReadFromMemory()) + if (Inst->mayReadFromMemory() && !TTI.canAlwaysSinkRead(Inst, SuccToSinkTo)) return false; // We don't want to sink across a critical edge if we don't dominate the @@ -101,7 +103,8 @@ /// instruction out of its current block into a successor. static bool SinkInstruction(Instruction *Inst, SmallPtrSetImpl &Stores, - DominatorTree &DT, LoopInfo &LI, AAResults &AA) { + DominatorTree &DT, LoopInfo &LI, AAResults &AA, + TargetTransformInfo &TTI) { // Don't sink static alloca instructions. CodeGen assumes allocas outside the // entry block are dynamically sized stack objects. @@ -152,7 +155,7 @@ // The nearest common dominator may be in a parent loop of BB, which may not // be beneficial. Find an ancestor. while (SuccToSinkTo != BB && - !IsAcceptableTarget(Inst, SuccToSinkTo, DT, LI)) + !IsAcceptableTarget(Inst, SuccToSinkTo, DT, LI, TTI)) SuccToSinkTo = DT.getNode(SuccToSinkTo)->getIDom()->getBlock(); if (SuccToSinkTo == BB) SuccToSinkTo = nullptr; @@ -172,7 +175,7 @@ } static bool ProcessBlock(BasicBlock &BB, DominatorTree &DT, LoopInfo &LI, - AAResults &AA) { + AAResults &AA, TargetTransformInfo &TTI) { // Can't sink anything out of a block that has less than two successors. if (BB.getTerminator()->getNumSuccessors() <= 1) return false; @@ -200,7 +203,7 @@ if (Inst->isDebugOrPseudoInst()) continue; - if (SinkInstruction(Inst, Stores, DT, LI, AA)) { + if (SinkInstruction(Inst, Stores, DT, LI, AA, TTI)) { ++NumSunk; MadeChange = true; } @@ -212,7 +215,8 @@ } static bool iterativelySinkInstructions(Function &F, DominatorTree &DT, - LoopInfo &LI, AAResults &AA) { + LoopInfo &LI, AAResults &AA, + TargetTransformInfo &TTI) { bool MadeChange, EverMadeChange = false; do { @@ -220,7 +224,7 @@ LLVM_DEBUG(dbgs() << "Sinking iteration " << NumSinkIter << "\n"); // Process all basic blocks. for (BasicBlock &I : F) - MadeChange |= ProcessBlock(I, DT, LI, AA); + MadeChange |= ProcessBlock(I, DT, LI, AA, TTI); EverMadeChange |= MadeChange; NumSinkIter++; } while (MadeChange); @@ -232,8 +236,9 @@ auto &DT = AM.getResult(F); auto &LI = AM.getResult(F); auto &AA = AM.getResult(F); + auto &TTI = AM.getResult(F); - if (!iterativelySinkInstructions(F, DT, LI, AA)) + if (!iterativelySinkInstructions(F, DT, LI, AA, TTI)) return PreservedAnalyses::all(); PreservedAnalyses PA; @@ -253,8 +258,9 @@ auto &DT = getAnalysis().getDomTree(); auto &LI = getAnalysis().getLoopInfo(); auto &AA = getAnalysis().getAAResults(); + auto &TTI = getAnalysis().getTTI(F); - return iterativelySinkInstructions(F, DT, LI, AA); + return iterativelySinkInstructions(F, DT, LI, AA, TTI); } void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -263,6 +269,7 @@ AU.addRequired(); AU.addRequired(); AU.addRequired(); + AU.addRequired(); AU.addPreserved(); AU.addPreserved(); } @@ -271,6 +278,7 @@ char SinkingLegacyPass::ID = 0; INITIALIZE_PASS_BEGIN(SinkingLegacyPass, "sink", "Code sinking", false, false) +INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)