Index: include/llvm/InitializePasses.h =================================================================== --- include/llvm/InitializePasses.h +++ include/llvm/InitializePasses.h @@ -174,6 +174,7 @@ void initializeLCSSAVerificationPassPass(PassRegistry&); void initializeLCSSAWrapperPassPass(PassRegistry&); void initializeLateCFGSimplifyPassPass(PassRegistry&); +void initializeLateJumpThreadingPass(PassRegistry&); void initializeLazyBlockFrequencyInfoPassPass(PassRegistry&); void initializeLazyBranchProbabilityInfoPassPass(PassRegistry&); void initializeLazyMachineBlockFrequencyInfoPassPass(PassRegistry&); Index: include/llvm/Transforms/Scalar.h =================================================================== --- include/llvm/Transforms/Scalar.h +++ include/llvm/Transforms/Scalar.h @@ -251,6 +251,7 @@ // internal BB duplication default threshold. // FunctionPass *createJumpThreadingPass(int Threshold = -1); +FunctionPass *createLateJumpThreadingPass(int Threshold = -1); //===----------------------------------------------------------------------===// // Index: include/llvm/Transforms/Scalar/JumpThreading.h =================================================================== --- include/llvm/Transforms/Scalar/JumpThreading.h +++ include/llvm/Transforms/Scalar/JumpThreading.h @@ -66,6 +66,7 @@ std::unique_ptr BPI; bool HasProfileData = false; bool HasGuards = false; + bool IsLate = false; #ifdef NDEBUG SmallPtrSet LoopHeaders; #else @@ -88,7 +89,7 @@ }; public: - JumpThreadingPass(int T = -1); + JumpThreadingPass(int T = -1, bool Late = false); // Glue for old PM. bool runImpl(Function &F, TargetLibraryInfo *TLI_, LazyValueInfo *LVI_, @@ -139,6 +140,11 @@ bool doesBlockHaveProfileData(BasicBlock *BB); }; +class LateJumpThreadingPass : public JumpThreadingPass { +public: + LateJumpThreadingPass(int T = -1) : JumpThreadingPass(T, true) {} +}; + } // end namespace llvm #endif Index: lib/Passes/PassRegistry.def =================================================================== --- lib/Passes/PassRegistry.def +++ lib/Passes/PassRegistry.def @@ -169,6 +169,7 @@ FUNCTION_PASS("nary-reassociate", NaryReassociatePass()) FUNCTION_PASS("newgvn", NewGVNPass()) FUNCTION_PASS("jump-threading", JumpThreadingPass()) +FUNCTION_PASS("late-jump-threading", LateJumpThreadingPass()) FUNCTION_PASS("partially-inline-libcalls", PartiallyInlineLibCallsPass()) FUNCTION_PASS("lcssa", LCSSAPass()) FUNCTION_PASS("loop-data-prefetch", LoopDataPrefetchPass()) Index: lib/Transforms/Scalar/JumpThreading.cpp =================================================================== --- lib/Transforms/Scalar/JumpThreading.cpp +++ lib/Transforms/Scalar/JumpThreading.cpp @@ -91,7 +91,7 @@ public: static char ID; // Pass identification - JumpThreading(int T = -1) : FunctionPass(ID), Impl(T) { + JumpThreading(int T = -1) : FunctionPass(ID), Impl(T, false) { initializeJumpThreadingPass(*PassRegistry::getPassRegistry()); } @@ -108,6 +108,29 @@ void releaseMemory() override { Impl.releaseMemory(); } }; + + class LateJumpThreading : public FunctionPass { + JumpThreadingPass Impl; + + public: + static char ID; // Pass identification + LateJumpThreading(int T = -1) : FunctionPass(ID), Impl(T, true) { + initializeLateJumpThreadingPass(*PassRegistry::getPassRegistry()); + } + + bool runOnFunction(Function &F) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + if (PrintLVIAfterJumpThreading) + AU.addRequired(); + AU.addRequired(); + AU.addRequired(); + AU.addPreserved(); + AU.addRequired(); + } + + void releaseMemory() override { Impl.releaseMemory(); } + }; } char JumpThreading::ID = 0; @@ -119,10 +142,24 @@ INITIALIZE_PASS_END(JumpThreading, "jump-threading", "Jump Threading", false, false) +char LateJumpThreading::ID = 0; +INITIALIZE_PASS_BEGIN(LateJumpThreading, "late-jump-threading", + "Late Jump Threading", false, false) +INITIALIZE_PASS_DEPENDENCY(LazyValueInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) +INITIALIZE_PASS_END(LateJumpThreading, "late-jump-threading", + "Late Jump Threading", false, false) + // Public interface to the Jump Threading pass -FunctionPass *llvm::createJumpThreadingPass(int Threshold) { return new JumpThreading(Threshold); } +FunctionPass *llvm::createJumpThreadingPass(int Threshold) { + return new JumpThreading(Threshold); +} +FunctionPass *llvm::createLateJumpThreadingPass(int Threshold) { + return new LateJumpThreading(Threshold); +} -JumpThreadingPass::JumpThreadingPass(int T) { +JumpThreadingPass::JumpThreadingPass(int T, bool Late) : IsLate(Late) { BBDupThreshold = (T == -1) ? BBDuplicateThreshold : unsigned(T); } @@ -262,6 +299,31 @@ return Changed; } +bool LateJumpThreading::runOnFunction(Function &F) { + if (skipFunction(F)) + return false; + auto TLI = &getAnalysis().getTLI(); + auto LVI = &getAnalysis().getLVI(); + auto AA = &getAnalysis().getAAResults(); + std::unique_ptr BFI; + std::unique_ptr BPI; + bool HasProfileData = F.getEntryCount().hasValue(); + if (HasProfileData) { + LoopInfo LI{DominatorTree(F)}; + BPI.reset(new BranchProbabilityInfo(F, LI, TLI)); + BFI.reset(new BlockFrequencyInfo(F, *BPI, LI)); + } + + bool Changed = Impl.runImpl(F, TLI, LVI, AA, HasProfileData, std::move(BFI), + std::move(BPI)); + if (PrintLVIAfterJumpThreading) { + dbgs() << "LVI for function '" << F.getName() << "':\n"; + LVI->printLVI(F, getAnalysis().getDomTree(), + dbgs()); + } + return Changed; +} + PreservedAnalyses JumpThreadingPass::run(Function &F, FunctionAnalysisManager &AM) { @@ -1791,7 +1853,7 @@ // If threading this would thread across a loop header, don't thread the edge. // See the comments above FindLoopHeaders for justifications and caveats. - if (LoopHeaders.count(BB) || LoopHeaders.count(SuccBB)) { + if (LoopHeaders.count(BB) || (!IsLate && LoopHeaders.count(SuccBB))) { DEBUG({ bool BBIsHeader = LoopHeaders.count(BB); bool SuccIsHeader = LoopHeaders.count(SuccBB); Index: lib/Transforms/Scalar/Scalar.cpp =================================================================== --- lib/Transforms/Scalar/Scalar.cpp +++ lib/Transforms/Scalar/Scalar.cpp @@ -55,6 +55,7 @@ initializeIndVarSimplifyLegacyPassPass(Registry); initializeInferAddressSpacesPass(Registry); initializeJumpThreadingPass(Registry); + initializeLateJumpThreadingPass(Registry); initializeLegacyLICMPassPass(Registry); initializeLegacyLoopSinkPassPass(Registry); initializeLoopDataPrefetchLegacyPassPass(Registry);