Index: llvm/trunk/include/llvm/InitializePasses.h =================================================================== --- llvm/trunk/include/llvm/InitializePasses.h +++ llvm/trunk/include/llvm/InitializePasses.h @@ -226,7 +226,7 @@ void initializeMemorySSAWrapperPassPass(PassRegistry&); void initializeMemorySanitizerPass(PassRegistry&); void initializeMergeFunctionsPass(PassRegistry&); -void initializeMergedLoadStoreMotionPass(PassRegistry &); +void initializeMergedLoadStoreMotionLegacyPassPass(PassRegistry &); void initializeMetaRenamerPass(PassRegistry&); void initializeModuleDebugInfoPrinterPass(PassRegistry&); void initializeModuleSummaryIndexWrapperPassPass(PassRegistry &); Index: llvm/trunk/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h =================================================================== --- llvm/trunk/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h +++ llvm/trunk/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h @@ -0,0 +1,39 @@ +//===- MergedLoadStoreMotion.h - merge and hoist/sink load/stores ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +//! \file +//! \brief This pass performs merges of loads and stores on both sides of a +// diamond (hammock). It hoists the loads and sinks the stores. +// +// The algorithm iteratively hoists two loads to the same address out of a +// diamond (hammock) and merges them into a single load in the header. Similar +// it sinks and merges two stores to the tail block (footer). The algorithm +// iterates over the instructions of one side of the diamond and attempts to +// find a matching load/store on the other side. It hoists / sinks when it +// thinks it safe to do so. This optimization helps with eg. hiding load +// latencies, triggering if-conversion, and reducing static code size. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_MERGEDLOADSTOREMOTION_H +#define LLVM_TRANSFORMS_SCALAR_MERGEDLOADSTOREMOTION_H + +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { +class MergedLoadStoreMotionPass + : public PassInfoMixin { +public: + PreservedAnalyses run(Function &F, AnalysisManager &AM); +}; + +} + +#endif // LLVM_TRANSFORMS_SCALAR_MERGEDLOADSTOREMOTION_H Index: llvm/trunk/lib/LTO/LTOCodeGenerator.cpp =================================================================== --- llvm/trunk/lib/LTO/LTOCodeGenerator.cpp +++ llvm/trunk/lib/LTO/LTOCodeGenerator.cpp @@ -123,7 +123,7 @@ initializeReversePostOrderFunctionAttrsLegacyPassPass(R); initializeGlobalsAAWrapperPassPass(R); initializeLICMPass(R); - initializeMergedLoadStoreMotionPass(R); + initializeMergedLoadStoreMotionLegacyPassPass(R); initializeGVNLegacyPassPass(R); initializeMemCpyOptLegacyPassPass(R); initializeDCELegacyPassPass(R); Index: llvm/trunk/lib/Passes/PassBuilder.cpp =================================================================== --- llvm/trunk/lib/Passes/PassBuilder.cpp +++ llvm/trunk/lib/Passes/PassBuilder.cpp @@ -83,6 +83,7 @@ #include "llvm/Transforms/Scalar/LowerAtomic.h" #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h" #include "llvm/Transforms/Scalar/MemCpyOptimizer.h" +#include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h" #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" #include "llvm/Transforms/Scalar/Reassociate.h" #include "llvm/Transforms/Scalar/SCCP.h" Index: llvm/trunk/lib/Passes/PassRegistry.def =================================================================== --- llvm/trunk/lib/Passes/PassRegistry.def +++ llvm/trunk/lib/Passes/PassRegistry.def @@ -134,6 +134,7 @@ FUNCTION_PASS("gvn", GVN()) FUNCTION_PASS("mem2reg", PromotePass()) FUNCTION_PASS("memcpyopt", MemCpyOptPass()) +FUNCTION_PASS("mldst-motion", MergedLoadStoreMotionPass()) FUNCTION_PASS("jump-threading", JumpThreadingPass()) FUNCTION_PASS("partially-inline-libcalls", PartiallyInlineLibCallsPass()) FUNCTION_PASS("lcssa", LCSSAPass()) Index: llvm/trunk/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp =================================================================== --- llvm/trunk/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp +++ llvm/trunk/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp @@ -72,6 +72,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/CFG.h" @@ -95,32 +96,20 @@ //===----------------------------------------------------------------------===// // MergedLoadStoreMotion Pass //===----------------------------------------------------------------------===// +class MergedLoadStoreMotion { + MemoryDependenceResults *MD = nullptr; + AliasAnalysis *AA = nullptr; -namespace { -class MergedLoadStoreMotion : public FunctionPass { - AliasAnalysis *AA; - MemoryDependenceResults *MD; + // The mergeLoad/Store algorithms could have Size0 * Size1 complexity, + // where Size0 and Size1 are the #instructions on the two sides of + // the diamond. The constant chosen here is arbitrary. Compiler Time + // Control is enforced by the check Size0 * Size1 < MagicCompileTimeControl. + const int MagicCompileTimeControl = 250; public: - static char ID; // Pass identification, replacement for typeid - MergedLoadStoreMotion() - : FunctionPass(ID), MD(nullptr), MagicCompileTimeControl(250) { - initializeMergedLoadStoreMotionPass(*PassRegistry::getPassRegistry()); - } - - bool runOnFunction(Function &F) override; + bool run(Function &F, MemoryDependenceResults *MD, AliasAnalysis &AA); private: - // This transformation requires dominator postdominator info - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.setPreservesCFG(); - AU.addRequired(); - AU.addPreserved(); - AU.addPreserved(); - } - - // Helper routines - /// /// \brief Remove instruction from parent and update memory dependence /// analysis. @@ -145,30 +134,8 @@ const Instruction &End, MemoryLocation Loc); bool sinkStore(BasicBlock *BB, StoreInst *SinkCand, StoreInst *ElseInst); bool mergeStores(BasicBlock *BB); - // The mergeLoad/Store algorithms could have Size0 * Size1 complexity, - // where Size0 and Size1 are the #instructions on the two sides of - // the diamond. The constant chosen here is arbitrary. Compiler Time - // Control is enforced by the check Size0 * Size1 < MagicCompileTimeControl. - const int MagicCompileTimeControl; }; -char MergedLoadStoreMotion::ID = 0; -} // anonymous namespace - -/// -/// \brief createMergedLoadStoreMotionPass - The public interface to this file. -/// -FunctionPass *llvm::createMergedLoadStoreMotionPass() { - return new MergedLoadStoreMotion(); -} - -INITIALIZE_PASS_BEGIN(MergedLoadStoreMotion, "mldst-motion", - "MergedLoadStoreMotion", false, false) -INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass) -INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) -INITIALIZE_PASS_END(MergedLoadStoreMotion, "mldst-motion", - "MergedLoadStoreMotion", false, false) - /// /// \brief Remove instruction from parent and update memory dependence analysis. /// @@ -552,16 +519,10 @@ return MergedStores; } -/// -/// \brief Run the transformation for each function -/// -bool MergedLoadStoreMotion::runOnFunction(Function &F) { - if (skipFunction(F)) - return false; - - auto *MDWP = getAnalysisIfAvailable(); - MD = MDWP ? &MDWP->getMemDep() : nullptr; - AA = &getAnalysis().getAAResults(); +bool MergedLoadStoreMotion::run(Function &F, MemoryDependenceResults *MD, + AliasAnalysis &AA) { + this->MD = MD; + this->AA = &AA; bool Changed = false; DEBUG(dbgs() << "Instruction Merger\n"); @@ -580,3 +541,67 @@ } return Changed; } + +namespace { +class MergedLoadStoreMotionLegacyPass : public FunctionPass { +public: + static char ID; // Pass identification, replacement for typeid + MergedLoadStoreMotionLegacyPass() : FunctionPass(ID) { + initializeMergedLoadStoreMotionLegacyPassPass( + *PassRegistry::getPassRegistry()); + } + + /// + /// \brief Run the transformation for each function + /// + bool runOnFunction(Function &F) override { + if (skipFunction(F)) + return false; + MergedLoadStoreMotion Impl; + auto *MDWP = getAnalysisIfAvailable(); + return Impl.run(F, MDWP ? &MDWP->getMemDep() : nullptr, + getAnalysis().getAAResults()); + } + +private: + // This transformation requires dominator postdominator info + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + AU.addRequired(); + AU.addPreserved(); + AU.addPreserved(); + } +}; + +char MergedLoadStoreMotionLegacyPass::ID = 0; +} // anonymous namespace + +/// +/// \brief createMergedLoadStoreMotionPass - The public interface to this file. +/// +FunctionPass *llvm::createMergedLoadStoreMotionPass() { + return new MergedLoadStoreMotionLegacyPass(); +} + +INITIALIZE_PASS_BEGIN(MergedLoadStoreMotionLegacyPass, "mldst-motion", + "MergedLoadStoreMotion", false, false) +INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass) +INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) +INITIALIZE_PASS_END(MergedLoadStoreMotionLegacyPass, "mldst-motion", + "MergedLoadStoreMotion", false, false) + +PreservedAnalyses +MergedLoadStoreMotionPass::run(Function &F, AnalysisManager &AM) { + MergedLoadStoreMotion Impl; + auto *MD = AM.getCachedResult(F); + auto &AA = AM.getResult(F); + if (!Impl.run(F, MD, AA)) + return PreservedAnalyses::all(); + + // FIXME: This pass should also 'preserve the CFG'. + // The new pass manager has currently no way to do it. + PreservedAnalyses PA; + PA.preserve(); + PA.preserve(); + return PA; +} Index: llvm/trunk/lib/Transforms/Scalar/Scalar.cpp =================================================================== --- llvm/trunk/lib/Transforms/Scalar/Scalar.cpp +++ llvm/trunk/lib/Transforms/Scalar/Scalar.cpp @@ -65,7 +65,7 @@ initializeLowerExpectIntrinsicPass(Registry); initializeLowerGuardIntrinsicPass(Registry); initializeMemCpyOptLegacyPassPass(Registry); - initializeMergedLoadStoreMotionPass(Registry); + initializeMergedLoadStoreMotionLegacyPassPass(Registry); initializeNaryReassociatePass(Registry); initializePartiallyInlineLibCallsLegacyPassPass(Registry); initializeReassociateLegacyPassPass(Registry); Index: llvm/trunk/test/Transforms/InstMerge/exceptions.ll =================================================================== --- llvm/trunk/test/Transforms/InstMerge/exceptions.ll +++ llvm/trunk/test/Transforms/InstMerge/exceptions.ll @@ -1,4 +1,7 @@ ; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s +; RUN: opt -aa-pipeline=basic-aa -passes='require',mldst-motion \ +; RUN: -S < %s | FileCheck %s + target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu"