diff --git a/llvm/include/llvm-c/Transforms/Coroutines.h b/llvm/include/llvm-c/Transforms/Coroutines.h --- a/llvm/include/llvm-c/Transforms/Coroutines.h +++ b/llvm/include/llvm-c/Transforms/Coroutines.h @@ -37,7 +37,7 @@ /** See llvm::createCoroSplitLegacyPass function. */ void LLVMAddCoroSplitPass(LLVMPassManagerRef PM); -/** See llvm::createCoroElidePass function. */ +/** See llvm::createCoroElideLegacyPass function. */ void LLVMAddCoroElidePass(LLVMPassManagerRef PM); /** See llvm::createCoroCleanupPass function. */ diff --git a/llvm/include/llvm/Transforms/Coroutines.h b/llvm/include/llvm/Transforms/Coroutines.h --- a/llvm/include/llvm/Transforms/Coroutines.h +++ b/llvm/include/llvm/Transforms/Coroutines.h @@ -27,7 +27,7 @@ /// Analyze coroutines use sites, devirtualize resume/destroy calls and elide /// heap allocation for coroutine frame where possible. -Pass *createCoroElidePass(); +Pass *createCoroElideLegacyPass(); /// Lower all remaining coroutine intrinsics. Pass *createCoroCleanupPass(); diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroElide.h b/llvm/include/llvm/Transforms/Coroutines/CoroElide.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Transforms/Coroutines/CoroElide.h @@ -0,0 +1,30 @@ +//===---- CoroElide.h - Coroutine frame allocation elision ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// \file +// This file declares a pass that replaces dynamic allocation of coroutine +// frames with alloca and replaces calls to llvm.coro.resume and +// llvm.coro.destroy with direct calls to coroutine sub-functions. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_COROUTINES_COROELIDE_H +#define LLVM_TRANSFORMS_COROUTINES_COROELIDE_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class Function; + +struct CoroElidePass : PassInfoMixin { + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_COROUTINES_COROELIDE_H diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -68,6 +68,7 @@ #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h" #include "llvm/Transforms/Coroutines/CoroEarly.h" +#include "llvm/Transforms/Coroutines/CoroElide.h" #include "llvm/Transforms/Coroutines/CoroSplit.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/ArgumentPromotion.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -168,6 +168,7 @@ FUNCTION_PASS("consthoist", ConstantHoistingPass()) FUNCTION_PASS("chr", ControlHeightReductionPass()) FUNCTION_PASS("coro-early", CoroEarlyPass()) +FUNCTION_PASS("coro-elide", CoroElidePass()) FUNCTION_PASS("correlated-propagation", CorrelatedValuePropagationPass()) FUNCTION_PASS("dce", DCEPass()) FUNCTION_PASS("div-rem-pairs", DivRemPairsPass()) diff --git a/llvm/lib/Transforms/Coroutines/CoroElide.cpp b/llvm/lib/Transforms/Coroutines/CoroElide.cpp --- a/llvm/lib/Transforms/Coroutines/CoroElide.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroElide.cpp @@ -5,11 +5,8 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// This pass replaces dynamic allocation of coroutine frame with alloca and -// replaces calls to llvm.coro.resume and llvm.coro.destroy with direct calls -// to coroutine sub-functions. -//===----------------------------------------------------------------------===// +#include "llvm/Transforms/Coroutines/CoroElide.h" #include "CoroInternal.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/InstructionSimplify.h" @@ -24,7 +21,7 @@ #define DEBUG_TYPE "coro-elide" namespace { -// Created on demand if CoroElide pass has work to do. +// Created on demand if the coro-elide pass has work to do. struct Lowerer : coro::LowererBase { SmallVector CoroIds; SmallVector CoroBegins; @@ -37,6 +34,7 @@ void elideHeapAllocations(Function *F, Type *FrameTy, AAResults &AA); bool shouldElide(Function *F, DominatorTree &DT) const; + void collectPostSplitCoroIds(Function *F); bool processCoroId(CoroIdInst *, AAResults &AA, DominatorTree &DT); }; } // end anonymous namespace @@ -188,6 +186,16 @@ return ReferencedCoroBegins.size() == CoroBegins.size(); } +void Lowerer::collectPostSplitCoroIds(Function *F) { + CoroIds.clear(); + for (auto &I : instructions(F)) + if (auto *CII = dyn_cast(&I)) + if (CII->getInfo().isPostSplit()) + // If it is the coroutine itself, don't touch it. + if (CII->getCoroutine() != CII->getFunction()) + CoroIds.push_back(CII); +} + bool Lowerer::processCoroId(CoroIdInst *CoroId, AAResults &AA, DominatorTree &DT) { CoroBegins.clear(); @@ -272,21 +280,47 @@ return true; } -//===----------------------------------------------------------------------===// -// Top Level Driver -//===----------------------------------------------------------------------===// +static bool declaresCoroElideIntrinsics(Module &M) { + return coro::declaresIntrinsics(M, {"llvm.coro.id"}); +} + +PreservedAnalyses CoroElidePass::run(Function &F, FunctionAnalysisManager &AM) { + auto &M = *F.getParent(); + if (!declaresCoroElideIntrinsics(M)) + return PreservedAnalyses::all(); + + bool Changed = false; + + if (F.hasFnAttribute(CORO_PRESPLIT_ATTR)) + Changed = replaceDevirtTrigger(F); + + Lowerer L(M); + L.CoroIds.clear(); + L.collectPostSplitCoroIds(&F); + // If we did not find any coro.id, there is nothing to do. + if (L.CoroIds.empty()) + return PreservedAnalyses::all(); + + AAResults &AA = AM.getResult(F); + DominatorTree &DT = AM.getResult(F); + + for (auto *CII : L.CoroIds) + Changed |= L.processCoroId(CII, AA, DT); + + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} namespace { -struct CoroElide : FunctionPass { +struct CoroElideLegacy : FunctionPass { static char ID; - CoroElide() : FunctionPass(ID) { - initializeCoroElidePass(*PassRegistry::getPassRegistry()); + CoroElideLegacy() : FunctionPass(ID) { + initializeCoroElideLegacyPass(*PassRegistry::getPassRegistry()); } std::unique_ptr L; bool doInitialization(Module &M) override { - if (coro::declaresIntrinsics(M, {"llvm.coro.id"})) + if (declaresCoroElideIntrinsics(M)) L = std::make_unique(M); return false; } @@ -301,15 +335,7 @@ Changed = replaceDevirtTrigger(F); L->CoroIds.clear(); - - // Collect all PostSplit coro.ids. - for (auto &I : instructions(F)) - if (auto *CII = dyn_cast(&I)) - if (CII->getInfo().isPostSplit()) - // If it is the coroutine itself, don't touch it. - if (CII->getCoroutine() != CII->getFunction()) - L->CoroIds.push_back(CII); - + L->collectPostSplitCoroIds(&F); // If we did not find any coro.id, there is nothing to do. if (L->CoroIds.empty()) return Changed; @@ -330,15 +356,15 @@ }; } -char CoroElide::ID = 0; +char CoroElideLegacy::ID = 0; INITIALIZE_PASS_BEGIN( - CoroElide, "coro-elide", + CoroElideLegacy, "coro-elide", "Coroutine frame allocation elision and indirect calls replacement", false, false) INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) INITIALIZE_PASS_END( - CoroElide, "coro-elide", + CoroElideLegacy, "coro-elide", "Coroutine frame allocation elision and indirect calls replacement", false, false) -Pass *llvm::createCoroElidePass() { return new CoroElide(); } +Pass *llvm::createCoroElideLegacyPass() { return new CoroElideLegacy(); } diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h --- a/llvm/lib/Transforms/Coroutines/CoroInternal.h +++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h @@ -23,7 +23,7 @@ void initializeCoroEarlyLegacyPass(PassRegistry &); void initializeCoroSplitLegacyPass(PassRegistry &); -void initializeCoroElidePass(PassRegistry &); +void initializeCoroElideLegacyPass(PassRegistry &); void initializeCoroCleanupPass(PassRegistry &); // CoroEarly pass marks every function that has coro.begin with a string diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp --- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp +++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp @@ -45,14 +45,14 @@ void llvm::initializeCoroutines(PassRegistry &Registry) { initializeCoroEarlyLegacyPass(Registry); initializeCoroSplitLegacyPass(Registry); - initializeCoroElidePass(Registry); + initializeCoroElideLegacyPass(Registry); initializeCoroCleanupPass(Registry); } static void addCoroutineOpt0Passes(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { PM.add(createCoroSplitLegacyPass()); - PM.add(createCoroElidePass()); + PM.add(createCoroElideLegacyPass()); PM.add(createBarrierNoopPass()); PM.add(createCoroCleanupPass()); @@ -65,7 +65,7 @@ static void addCoroutineScalarOptimizerPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { - PM.add(createCoroElidePass()); + PM.add(createCoroElideLegacyPass()); } static void addCoroutineSCCPasses(const PassManagerBuilder &Builder, @@ -643,7 +643,7 @@ } void LLVMAddCoroElidePass(LLVMPassManagerRef PM) { - unwrap(PM)->add(createCoroElidePass()); + unwrap(PM)->add(createCoroElideLegacyPass()); } void LLVMAddCoroCleanupPass(LLVMPassManagerRef PM) { diff --git a/llvm/test/Transforms/Coroutines/coro-elide.ll b/llvm/test/Transforms/Coroutines/coro-elide.ll --- a/llvm/test/Transforms/Coroutines/coro-elide.ll +++ b/llvm/test/Transforms/Coroutines/coro-elide.ll @@ -1,6 +1,7 @@ ; Tests that the coro.destroy and coro.resume are devirtualized where possible, ; SCC pipeline restarts and inlines the direct calls. ; RUN: opt < %s -S -inline -coro-elide -dce | FileCheck %s +; RUN: opt < %s -S -passes='cgscc(repeat<2>(inline,function(coro-elide,dce)))' | FileCheck %s declare void @print(i32) nounwind diff --git a/llvm/test/Transforms/Coroutines/coro-heap-elide.ll b/llvm/test/Transforms/Coroutines/coro-heap-elide.ll --- a/llvm/test/Transforms/Coroutines/coro-heap-elide.ll +++ b/llvm/test/Transforms/Coroutines/coro-heap-elide.ll @@ -2,6 +2,9 @@ ; elided and any tail calls referencing the coroutine frame has the tail ; call attribute removed. ; RUN: opt < %s -S -inline -coro-elide -instsimplify -simplifycfg | FileCheck %s +; RUN: opt < %s -S \ +; RUN: -passes='cgscc(repeat<2>(inline,function(coro-elide,instsimplify,simplify-cfg)))' \ +; RUN: -aa-pipeline='basic-aa' | FileCheck %s declare void @print(i32) nounwind