Index: llvm/include/llvm/Analysis/TargetTransformInfo.h =================================================================== --- llvm/include/llvm/Analysis/TargetTransformInfo.h +++ llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -1503,6 +1503,10 @@ bool hasActiveVectorLength(unsigned Opcode, Type *DataType, Align Alignment) const; + /// \returns True when division operations are allowed to be executed speculatively + /// unconditionally. + bool enableUncondDivisionSpeculation() const; + struct VPLegalization { enum VPTransform { // keep the predicating parameter @@ -1885,6 +1889,7 @@ virtual bool supportsScalableVectors() const = 0; virtual bool hasActiveVectorLength(unsigned Opcode, Type *DataType, Align Alignment) const = 0; + virtual bool enableUncondDivisionSpeculation() const = 0; virtual VPLegalization getVPLegalizationStrategy(const VPIntrinsic &PI) const = 0; }; @@ -2546,6 +2551,10 @@ return Impl.hasActiveVectorLength(Opcode, DataType, Alignment); } + bool enableUncondDivisionSpeculation() const override { + return Impl.enableUncondDivisionSpeculation(); + } + VPLegalization getVPLegalizationStrategy(const VPIntrinsic &PI) const override { return Impl.getVPLegalizationStrategy(PI); Index: llvm/include/llvm/Analysis/TargetTransformInfoImpl.h =================================================================== --- llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -841,6 +841,8 @@ return false; } + bool enableUncondDivisionSpeculation() const { return false; } + TargetTransformInfo::VPLegalization getVPLegalizationStrategy(const VPIntrinsic &PI) const { return TargetTransformInfo::VPLegalization( Index: llvm/include/llvm/Analysis/ValueTracking.h =================================================================== --- llvm/include/llvm/Analysis/ValueTracking.h +++ llvm/include/llvm/Analysis/ValueTracking.h @@ -42,6 +42,7 @@ class OptimizationRemarkEmitter; class StringRef; class TargetLibraryInfo; +class TargetTransformInfo; class Value; constexpr unsigned MaxAnalysisRecursionDepth = 6; @@ -454,7 +455,8 @@ const Instruction *CtxI = nullptr, AssumptionCache *AC = nullptr, const DominatorTree *DT = nullptr, - const TargetLibraryInfo *TLI = nullptr); + const TargetLibraryInfo *TLI = nullptr, + const TargetTransformInfo *TTI = nullptr); /// This returns the same result as isSafeToSpeculativelyExecute if Opcode is /// the actual opcode of Inst. If the provided and actual opcode differ, the @@ -476,7 +478,8 @@ bool isSafeToSpeculativelyExecuteWithOpcode( unsigned Opcode, const Instruction *Inst, const Instruction *CtxI = nullptr, AssumptionCache *AC = nullptr, const DominatorTree *DT = nullptr, - const TargetLibraryInfo *TLI = nullptr); + const TargetLibraryInfo *TLI = nullptr, + const TargetTransformInfo *TTI = nullptr); /// Returns true if the result or effects of the given instructions \p I /// depend values not reachable through the def use graph. Index: llvm/include/llvm/Transforms/Utils/LoopUtils.h =================================================================== --- llvm/include/llvm/Transforms/Utils/LoopUtils.h +++ llvm/include/llvm/Transforms/Utils/LoopUtils.h @@ -171,8 +171,8 @@ /// \p AllowSpeculation is whether values should be hoisted even if they are not /// guaranteed to execute in the loop, but are safe to speculatively execute. bool hoistRegion(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *, - AssumptionCache *, TargetLibraryInfo *, Loop *, - MemorySSAUpdater &, ScalarEvolution *, ICFLoopSafetyInfo *, + AssumptionCache *, TargetLibraryInfo *, TargetTransformInfo *, + Loop *, MemorySSAUpdater &, ScalarEvolution *, ICFLoopSafetyInfo *, SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *, bool, bool AllowSpeculation); @@ -210,7 +210,8 @@ const SmallSetVector &, SmallVectorImpl &, SmallVectorImpl &, SmallVectorImpl &, PredIteratorCache &, LoopInfo *, DominatorTree *, AssumptionCache *AC, - const TargetLibraryInfo *, Loop *, MemorySSAUpdater &, ICFLoopSafetyInfo *, + const TargetLibraryInfo *, const TargetTransformInfo *TTI, Loop *, + MemorySSAUpdater &, ICFLoopSafetyInfo *, OptimizationRemarkEmitter *, bool AllowSpeculation); /// Does a BFS from a given node to all of its children inside a given loop. Index: llvm/lib/Analysis/TargetTransformInfo.cpp =================================================================== --- llvm/lib/Analysis/TargetTransformInfo.cpp +++ llvm/lib/Analysis/TargetTransformInfo.cpp @@ -1171,6 +1171,10 @@ return TTIImpl->hasActiveVectorLength(Opcode, DataType, Alignment); } +bool TargetTransformInfo::enableUncondDivisionSpeculation() const { + return TTIImpl->enableUncondDivisionSpeculation(); +} + TargetTransformInfo::Concept::~Concept() = default; TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {} Index: llvm/lib/Analysis/ValueTracking.cpp =================================================================== --- llvm/lib/Analysis/ValueTracking.cpp +++ llvm/lib/Analysis/ValueTracking.cpp @@ -34,6 +34,7 @@ #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/Argument.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" @@ -4734,15 +4735,16 @@ const Instruction *CtxI, AssumptionCache *AC, const DominatorTree *DT, - const TargetLibraryInfo *TLI) { + const TargetLibraryInfo *TLI, + const TargetTransformInfo *TTI) { return isSafeToSpeculativelyExecuteWithOpcode(Inst->getOpcode(), Inst, CtxI, - AC, DT, TLI); + AC, DT, TLI,TTI); } bool llvm::isSafeToSpeculativelyExecuteWithOpcode( unsigned Opcode, const Instruction *Inst, const Instruction *CtxI, AssumptionCache *AC, const DominatorTree *DT, - const TargetLibraryInfo *TLI) { + const TargetLibraryInfo *TLI,const TargetTransformInfo *TTI) { #ifndef NDEBUG if (Inst->getOpcode() != Opcode) { // Check that the operands are actually compatible with the Opcode override. @@ -4763,11 +4765,14 @@ } #endif + // Instructions are to be executed speculatively unconditionally. + bool ignoreDivCheck = TTI && TTI->enableUncondDivisionSpeculation(); switch (Opcode) { default: return true; case Instruction::UDiv: case Instruction::URem: { + if(ignoreDivCheck) return true; // x / y is undefined if y == 0. const APInt *V; if (match(Inst->getOperand(1), m_APInt(V))) @@ -4776,6 +4781,7 @@ } case Instruction::SDiv: case Instruction::SRem: { + if(ignoreDivCheck) return true; // x / y is undefined if y == 0 or x == INT_MIN and y == -1 const APInt *Numerator, *Denominator; if (!match(Inst->getOperand(1), m_APInt(Denominator))) Index: llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h =================================================================== --- llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h +++ llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h @@ -142,6 +142,7 @@ const ArrayRef &Types) const; bool hasActiveVectorLength(unsigned Opcode, Type *DataType, Align Alignment) const; + bool enableUncondDivisionSpeculation() const; InstructionCost getVPMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, Index: llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp =================================================================== --- llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp +++ llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp @@ -1415,6 +1415,8 @@ return IntWidth == 8 || IntWidth == 16 || IntWidth == 32 || IntWidth == 64; } +bool PPCTTIImpl::enableUncondDivisionSpeculation() const { return true; } + InstructionCost PPCTTIImpl::getVPMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, Index: llvm/lib/Transforms/Scalar/LICM.cpp =================================================================== --- llvm/lib/Transforms/Scalar/LICM.cpp +++ llvm/lib/Transforms/Scalar/LICM.cpp @@ -154,9 +154,9 @@ MemorySSAUpdater &MSSAU, OptimizationRemarkEmitter *ORE); static bool isSafeToExecuteUnconditionally( Instruction &Inst, const DominatorTree *DT, const TargetLibraryInfo *TLI, - const Loop *CurLoop, const LoopSafetyInfo *SafetyInfo, - OptimizationRemarkEmitter *ORE, const Instruction *CtxI, - AssumptionCache *AC, bool AllowSpeculation); + const TargetTransformInfo *TTI, const Loop *CurLoop, + const LoopSafetyInfo *SafetyInfo, OptimizationRemarkEmitter *ORE, + const Instruction *CtxI, AssumptionCache *AC, bool AllowSpeculation); static bool pointerInvalidatedByLoop(MemorySSA *MSSA, MemoryUse *MU, Loop *CurLoop, Instruction &I, SinkAndHoistLICMFlags &Flags); @@ -445,8 +445,8 @@ MSSAU, &SafetyInfo, Flags, ORE); Flags.setIsSink(false); if (Preheader) - Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, AC, TLI, L, - MSSAU, SE, &SafetyInfo, Flags, ORE, LoopNestMode, + Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, AC, TLI, TTI, + L, MSSAU, SE, &SafetyInfo, Flags, ORE, LoopNestMode, LicmAllowSpeculation); // Now that all loop invariants have been removed from the loop, promote any @@ -489,7 +489,7 @@ collectPromotionCandidates(MSSA, AA, L)) { LocalPromoted |= promoteLoopAccessesToScalars( PointerMustAliases, ExitBlocks, InsertPts, MSSAInsertPts, PIC, LI, - DT, AC, TLI, L, MSSAU, &SafetyInfo, ORE, LicmAllowSpeculation); + DT, AC, TLI, TTI, L, MSSAU, &SafetyInfo, ORE, LicmAllowSpeculation); } Promoted |= LocalPromoted; } while (LocalPromoted); @@ -849,8 +849,8 @@ /// bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC, - TargetLibraryInfo *TLI, Loop *CurLoop, - MemorySSAUpdater &MSSAU, ScalarEvolution *SE, + TargetLibraryInfo *TLI, TargetTransformInfo *TTI, + Loop *CurLoop, MemorySSAUpdater &MSSAU, ScalarEvolution *SE, ICFLoopSafetyInfo *SafetyInfo, SinkAndHoistLICMFlags &Flags, OptimizationRemarkEmitter *ORE, bool LoopNestMode, @@ -904,7 +904,7 @@ if (CurLoop->hasLoopInvariantOperands(&I) && canSinkOrHoistInst(I, AA, DT, CurLoop, MSSAU, true, Flags, ORE) && isSafeToExecuteUnconditionally( - I, DT, TLI, CurLoop, SafetyInfo, ORE, + I, DT, TLI, TTI, CurLoop, SafetyInfo, ORE, CurLoop->getLoopPreheader()->getTerminator(), AC, AllowSpeculation)) { hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, @@ -1743,7 +1743,8 @@ /// or if it is a trapping instruction and is guaranteed to execute. static bool isSafeToExecuteUnconditionally( Instruction &Inst, const DominatorTree *DT, const TargetLibraryInfo *TLI, - const Loop *CurLoop, const LoopSafetyInfo *SafetyInfo, + const TargetTransformInfo *TTI, const Loop *CurLoop, + const LoopSafetyInfo *SafetyInfo, OptimizationRemarkEmitter *ORE, const Instruction *CtxI, AssumptionCache *AC, bool AllowSpeculation) { if (AllowSpeculation && @@ -1937,9 +1938,9 @@ SmallVectorImpl &InsertPts, SmallVectorImpl &MSSAInsertPts, PredIteratorCache &PIC, LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC, - const TargetLibraryInfo *TLI, Loop *CurLoop, MemorySSAUpdater &MSSAU, - ICFLoopSafetyInfo *SafetyInfo, OptimizationRemarkEmitter *ORE, - bool AllowSpeculation) { + const TargetLibraryInfo *TLI, const TargetTransformInfo *TTI, Loop *CurLoop, + MemorySSAUpdater &MSSAU, ICFLoopSafetyInfo *SafetyInfo, + OptimizationRemarkEmitter *ORE, bool AllowSpeculation) { // Verify inputs. assert(LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && @@ -2054,7 +2055,7 @@ // alignment as well. if (!DereferenceableInPH || (InstAlignment > Alignment)) if (isSafeToExecuteUnconditionally( - *Load, DT, TLI, CurLoop, SafetyInfo, ORE, + *Load, DT, TLI, TTI, CurLoop, SafetyInfo, ORE, Preheader->getTerminator(), AC, AllowSpeculation)) { DereferenceableInPH = true; Alignment = std::max(Alignment, InstAlignment);