diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -255,7 +255,7 @@ void initializeLoopUnrollPass(PassRegistry&); void initializeLoopUnswitchPass(PassRegistry&); void initializeLoopVectorizePass(PassRegistry&); -void initializeLoopVersioningLICMPass(PassRegistry&); +void initializeLoopVersioningLICMLegacyPassPass(PassRegistry &); void initializeLoopVersioningLegacyPassPass(PassRegistry &); void initializeLowerAtomicLegacyPassPass(PassRegistry&); void initializeLowerConstantIntrinsicsPass(PassRegistry&); diff --git a/llvm/include/llvm/Transforms/Scalar/LoopVersioningLICM.h b/llvm/include/llvm/Transforms/Scalar/LoopVersioningLICM.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Transforms/Scalar/LoopVersioningLICM.h @@ -0,0 +1,25 @@ +//===- LoopVersioningLICM.h - LICM Loop Versioning ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_LOOPVERSIONINGLICM_H +#define LLVM_TRANSFORMS_SCALAR_LOOPVERSIONINGLICM_H + +#include "llvm/IR/PassManager.h" +#include "llvm/Transforms/Scalar/LoopPassManager.h" + +namespace llvm { + +class LoopVersioningLICMPass : public PassInfoMixin { +public: + PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &LAR, LPMUpdater &U); +}; + +} // namespace llvm + +#endif // LLVM_TRANSFORMS_SCALAR_LOOPVERSIONINGLICM_H diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -166,6 +166,7 @@ #include "llvm/Transforms/Scalar/LoopStrengthReduce.h" #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h" #include "llvm/Transforms/Scalar/LoopUnrollPass.h" +#include "llvm/Transforms/Scalar/LoopVersioningLICM.h" #include "llvm/Transforms/Scalar/LowerAtomic.h" #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h" #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -380,6 +380,7 @@ LOOP_PASS("guard-widening", GuardWideningPass()) LOOP_PASS("simple-loop-unswitch", SimpleLoopUnswitchPass()) LOOP_PASS("loop-reroll", LoopRerollPass()) +LOOP_PASS("loop-versioning-licm", LoopVersioningLICMPass()) #undef LOOP_PASS #ifndef LOOP_PASS_WITH_PARAMS diff --git a/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp b/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp --- a/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp +++ b/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp @@ -59,6 +59,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Transforms/Scalar/LoopVersioningLICM.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/AliasAnalysis.h" @@ -114,17 +115,18 @@ namespace { -struct LoopVersioningLICM : public LoopPass { +struct LoopVersioningLICMLegacyPass : public LoopPass { static char ID; - LoopVersioningLICM() - : LoopPass(ID), LoopDepthThreshold(LVLoopDepthThreshold), - InvariantThreshold(LVInvarThreshold) { - initializeLoopVersioningLICMPass(*PassRegistry::getPassRegistry()); + LoopVersioningLICMLegacyPass() : LoopPass(ID) { + initializeLoopVersioningLICMLegacyPassPass( + *PassRegistry::getPassRegistry()); } bool runOnLoop(Loop *L, LPPassManager &LPM) override; + StringRef getPassName() const override { return "Loop Versioning for LICM"; } + void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); AU.addRequired(); @@ -138,13 +140,25 @@ AU.addPreserved(); AU.addRequired(); } +}; - StringRef getPassName() const override { return "Loop Versioning for LICM"; } +struct LoopVersioningLICM { + // We don't explicitly pass in LoopAccessInfo to the constructor since the + // loop versioning might return early due to instructions that are not safe + // for versioning. By passing the proxy instead the construction of + // LoopAccessInfo will take place only when it's necessary. + LoopVersioningLICM(AliasAnalysis *AA, ScalarEvolution *SE, + OptimizationRemarkEmitter *ORE, + function_ref GetLAI) + : AA(AA), SE(SE), GetLAI(GetLAI), + LoopDepthThreshold(LVLoopDepthThreshold), + InvariantThreshold(LVInvarThreshold), ORE(ORE) {} + + bool runOnLoop(Loop *L, LoopInfo *LI, DominatorTree *DT); void reset() { AA = nullptr; SE = nullptr; - LAA = nullptr; CurLoop = nullptr; LoadAndStoreCounter = 0; InvariantCounter = 0; @@ -169,12 +183,12 @@ // Current ScalarEvolution ScalarEvolution *SE = nullptr; - // Current LoopAccessAnalysis - LoopAccessLegacyAnalysis *LAA = nullptr; - // Current Loop's LoopAccessInfo const LoopAccessInfo *LAI = nullptr; + // Proxy for retrieving LoopAccessInfo. + function_ref GetLAI; + // The current loop we are working on. Loop *CurLoop = nullptr; @@ -400,8 +414,8 @@ return false; } } - // Get LoopAccessInfo from current loop. - LAI = &LAA->getInfo(CurLoop); + // Get LoopAccessInfo from current loop via the proxy. + LAI = &GetLAI(CurLoop); // Check LoopAccessInfo for need of runtime check. if (LAI->getRuntimePointerChecking()->getChecks().empty()) { LLVM_DEBUG(dbgs() << " LAA: Runtime check not found !!\n"); @@ -562,30 +576,38 @@ } } -bool LoopVersioningLICM::runOnLoop(Loop *L, LPPassManager &LPM) { +bool LoopVersioningLICMLegacyPass::runOnLoop(Loop *L, LPPassManager &LPM) { + if (skipLoop(L)) + return false; + + AliasAnalysis *AA = &getAnalysis().getAAResults(); + ScalarEvolution *SE = &getAnalysis().getSE(); + OptimizationRemarkEmitter *ORE = + &getAnalysis().getORE(); + LoopInfo *LI = &getAnalysis().getLoopInfo(); + DominatorTree *DT = &getAnalysis().getDomTree(); + + auto GetLAI = [&](Loop *L) -> const LoopAccessInfo & { + return getAnalysis().getInfo(L); + }; + + return LoopVersioningLICM(AA, SE, ORE, GetLAI).runOnLoop(L, LI, DT); +} + +bool LoopVersioningLICM::runOnLoop(Loop *L, LoopInfo *LI, DominatorTree *DT) { // This will automatically release all resources hold by the current // LoopVersioningLICM object. AutoResetter Resetter(*this); - if (skipLoop(L)) - return false; - // Do not do the transformation if disabled by metadata. if (hasLICMVersioningTransformation(L) & TM_Disable) return false; - // Get Analysis information. - AA = &getAnalysis().getAAResults(); - SE = &getAnalysis().getSE(); - LAA = &getAnalysis(); - ORE = &getAnalysis().getORE(); - LAI = nullptr; // Set Current Loop CurLoop = L; CurAST.reset(new AliasSetTracker(*AA)); // Loop over the body of this loop, construct AST. - LoopInfo *LI = &getAnalysis().getLoopInfo(); for (auto *Block : L->getBlocks()) { if (LI->getLoopFor(Block) == L) // Ignore blocks in subloop. CurAST->add(*Block); // Incorporate the specified basic block @@ -600,7 +622,6 @@ // Do loop versioning. // Create memcheck for memory accessed inside loop. // Clone original loop, and set blocks properly. - DominatorTree *DT = &getAnalysis().getDomTree(); LoopVersioning LVer(*LAI, LAI->getRuntimePointerChecking()->getChecks(), CurLoop, LI, DT, SE); LVer.versionLoop(); @@ -620,9 +641,9 @@ return Changed; } -char LoopVersioningLICM::ID = 0; +char LoopVersioningLICMLegacyPass::ID = 0; -INITIALIZE_PASS_BEGIN(LoopVersioningLICM, "loop-versioning-licm", +INITIALIZE_PASS_BEGIN(LoopVersioningLICMLegacyPass, "loop-versioning-licm", "Loop Versioning For LICM", false, false) INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) @@ -633,7 +654,31 @@ INITIALIZE_PASS_DEPENDENCY(LoopSimplify) INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass) INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass) -INITIALIZE_PASS_END(LoopVersioningLICM, "loop-versioning-licm", +INITIALIZE_PASS_END(LoopVersioningLICMLegacyPass, "loop-versioning-licm", "Loop Versioning For LICM", false, false) -Pass *llvm::createLoopVersioningLICMPass() { return new LoopVersioningLICM(); } +Pass *llvm::createLoopVersioningLICMPass() { + return new LoopVersioningLICMLegacyPass(); +} + +namespace llvm { + +PreservedAnalyses LoopVersioningLICMPass::run(Loop &L, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &LAR, + LPMUpdater &U) { + AliasAnalysis *AA = &LAR.AA; + ScalarEvolution *SE = &LAR.SE; + DominatorTree *DT = &LAR.DT; + LoopInfo *LI = &LAR.LI; + const Function *F = L.getHeader()->getParent(); + OptimizationRemarkEmitter ORE(F); + + auto GetLAI = [&](Loop *L) -> const LoopAccessInfo & { + return AM.getResult(*L, LAR); + }; + + if (!LoopVersioningLICM(AA, SE, &ORE, GetLAI).runOnLoop(&L, LI, DT)) + return PreservedAnalyses::all(); + return getLoopPassPreservedAnalyses(); +} +} // namespace llvm diff --git a/llvm/lib/Transforms/Scalar/Scalar.cpp b/llvm/lib/Transforms/Scalar/Scalar.cpp --- a/llvm/lib/Transforms/Scalar/Scalar.cpp +++ b/llvm/lib/Transforms/Scalar/Scalar.cpp @@ -76,7 +76,7 @@ initializeLoopUnrollAndJamPass(Registry); initializeLoopUnswitchPass(Registry); initializeWarnMissedTransformationsLegacyPass(Registry); - initializeLoopVersioningLICMPass(Registry); + initializeLoopVersioningLICMLegacyPassPass(Registry); initializeLoopIdiomRecognizeLegacyPassPass(Registry); initializeLowerAtomicLegacyPassPass(Registry); initializeLowerConstantIntrinsicsPass(Registry); diff --git a/llvm/test/Transforms/LoopVersioningLICM/convergent.ll b/llvm/test/Transforms/LoopVersioningLICM/convergent.ll --- a/llvm/test/Transforms/LoopVersioningLICM/convergent.ll +++ b/llvm/test/Transforms/LoopVersioningLICM/convergent.ll @@ -1,4 +1,5 @@ ; RUN: opt -S -loop-versioning-licm -licm-versioning-invariant-threshold=0 %s | FileCheck %s +; RUN: opt -S -passes='loop-versioning-licm' -licm-versioning-invariant-threshold=0 %s | FileCheck %s ; Make sure the convergent attribute is respected, and no condition is ; introduced diff --git a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll --- a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll +++ b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -O1 -S -loop-versioning-licm -licm -debug-only=loop-versioning-licm 2>&1 | FileCheck %s +; RUN: opt < %s -S -passes='function(loop-versioning-licm),function(licm)' --aa-pipeline=default -debug-only=loop-versioning-licm 2>&1 | FileCheck %s ; REQUIRES: asserts ; ; Test to confirm loop is a candidate for LoopVersioningLICM. diff --git a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM2.ll b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM2.ll --- a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM2.ll +++ b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM2.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -O1 -S -loop-versioning-licm -licm -debug-only=loop-versioning-licm -disable-loop-unrolling 2>&1 | FileCheck %s +; RUN: opt < %s -S -passes='function(loop-versioning-licm),function(licm)' --aa-pipeline=default -debug-only=loop-versioning-licm -disable-loop-unrolling 2>&1 | FileCheck %s ; REQUIRES: asserts ; ; Test to confirm loop is a good candidate for LoopVersioningLICM diff --git a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM3.ll b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM3.ll --- a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM3.ll +++ b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM3.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -O1 -S -loop-versioning-licm -debug-only=loop-versioning-licm 2>&1 | FileCheck %s +; RUN: opt < %s -S -passes='loop-versioning-licm' -debug-only=loop-versioning-licm 2>&1 | FileCheck %s ; REQUIRES: asserts ; ; Test to confirm loop is not a candidate for LoopVersioningLICM. diff --git a/llvm/test/Transforms/LoopVersioningLICM/metadata.ll b/llvm/test/Transforms/LoopVersioningLICM/metadata.ll --- a/llvm/test/Transforms/LoopVersioningLICM/metadata.ll +++ b/llvm/test/Transforms/LoopVersioningLICM/metadata.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -O1 -S -loop-versioning-licm -licm 2>&1 | FileCheck %s +; RUN: opt < %s -S -passes='loop-versioning-licm,licm' 2>&1 | FileCheck %s ; CHECK-LABEL: @without_metadata( define i32 @without_metadata(i32* nocapture %var1, i32* nocapture readnone %var2, i32* nocapture %var3, i32 %itr) #0 {