diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroElide.h b/llvm/include/llvm/Transforms/Coroutines/CoroElide.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Transforms/Coroutines/CoroElide.h @@ -0,0 +1,30 @@ +//===---- CoroElide.h - Coroutine frame allocation elision ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// \file +// This file declares a pass that replaces dynamic allocation of coroutine +// frames with alloca and replaces calls to llvm.coro.resume and +// llvm.coro.destroy with direct calls to coroutine sub-functions. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_COROUTINES_COROELIDE_H +#define LLVM_TRANSFORMS_COROUTINES_COROELIDE_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class Function; + +struct CoroElidePass : PassInfoMixin { + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_COROUTINES_COROELIDE_H diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -68,6 +68,7 @@ #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h" #include "llvm/Transforms/Coroutines/CoroEarly.h" +#include "llvm/Transforms/Coroutines/CoroElide.h" #include "llvm/Transforms/Coroutines/CoroSplit.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/ArgumentPromotion.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -172,6 +172,7 @@ FUNCTION_PASS("consthoist", ConstantHoistingPass()) FUNCTION_PASS("chr", ControlHeightReductionPass()) FUNCTION_PASS("coro-early", CoroEarlyPass()) +FUNCTION_PASS("coro-elide", CoroElidePass()) FUNCTION_PASS("correlated-propagation", CorrelatedValuePropagationPass()) FUNCTION_PASS("dce", DCEPass()) FUNCTION_PASS("div-rem-pairs", DivRemPairsPass()) diff --git a/llvm/lib/Transforms/Coroutines/CoroElide.cpp b/llvm/lib/Transforms/Coroutines/CoroElide.cpp --- a/llvm/lib/Transforms/Coroutines/CoroElide.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroElide.cpp @@ -5,11 +5,8 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// This pass replaces dynamic allocation of coroutine frame with alloca and -// replaces calls to llvm.coro.resume and llvm.coro.destroy with direct calls -// to coroutine sub-functions. -//===----------------------------------------------------------------------===// +#include "llvm/Transforms/Coroutines/CoroElide.h" #include "CoroInternal.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/InstructionSimplify.h" @@ -37,6 +34,7 @@ void elideHeapAllocations(Function *F, Type *FrameTy, AAResults &AA); bool shouldElide(Function *F, DominatorTree &DT) const; + void collectPostSplitCoroIds(Function *F); bool processCoroId(CoroIdInst *, AAResults &AA, DominatorTree &DT); }; } // end anonymous namespace @@ -188,6 +186,16 @@ return ReferencedCoroBegins.size() == CoroBegins.size(); } +void Lowerer::collectPostSplitCoroIds(Function *F) { + CoroIds.clear(); + for (auto &I : instructions(F)) + if (auto *CII = dyn_cast(&I)) + if (CII->getInfo().isPostSplit()) + // If it is the coroutine itself, don't touch it. + if (CII->getCoroutine() != CII->getFunction()) + CoroIds.push_back(CII); +} + bool Lowerer::processCoroId(CoroIdInst *CoroId, AAResults &AA, DominatorTree &DT) { CoroBegins.clear(); @@ -272,9 +280,31 @@ return true; } -//===----------------------------------------------------------------------===// -// Top Level Driver -//===----------------------------------------------------------------------===// +static bool declaresCoroElideIntrinsics(Module &M) { + return coro::declaresIntrinsics(M, {"llvm.coro.id"}); +} + +PreservedAnalyses CoroElidePass::run(Function &F, FunctionAnalysisManager &AM) { + auto &M = *F.getParent(); + if (!declaresCoroElideIntrinsics(M)) + return PreservedAnalyses::all(); + + Lowerer L(M); + L.CoroIds.clear(); + L.collectPostSplitCoroIds(&F); + // If we did not find any coro.id, there is nothing to do. + if (L.CoroIds.empty()) + return PreservedAnalyses::all(); + + AAResults &AA = AM.getResult(F); + DominatorTree &DT = AM.getResult(F); + + bool Changed = false; + for (auto *CII : L.CoroIds) + Changed |= L.processCoroId(CII, AA, DT); + + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} namespace { struct CoroElideLegacy : FunctionPass { @@ -286,7 +316,7 @@ std::unique_ptr L; bool doInitialization(Module &M) override { - if (coro::declaresIntrinsics(M, {"llvm.coro.id"})) + if (declaresCoroElideIntrinsics(M)) L = std::make_unique(M); return false; } @@ -301,15 +331,7 @@ Changed = replaceDevirtTrigger(F); L->CoroIds.clear(); - - // Collect all PostSplit coro.ids. - for (auto &I : instructions(F)) - if (auto *CII = dyn_cast(&I)) - if (CII->getInfo().isPostSplit()) - // If it is the coroutine itself, don't touch it. - if (CII->getCoroutine() != CII->getFunction()) - L->CoroIds.push_back(CII); - + L->collectPostSplitCoroIds(&F); // If we did not find any coro.id, there is nothing to do. if (L->CoroIds.empty()) return Changed; diff --git a/llvm/test/Transforms/Coroutines/coro-elide.ll b/llvm/test/Transforms/Coroutines/coro-elide.ll --- a/llvm/test/Transforms/Coroutines/coro-elide.ll +++ b/llvm/test/Transforms/Coroutines/coro-elide.ll @@ -1,6 +1,9 @@ ; Tests that the coro.destroy and coro.resume are devirtualized where possible, ; SCC pipeline restarts and inlines the direct calls. ; RUN: opt < %s -S -inline -coro-elide -dce | FileCheck %s +; RUN: opt < %s -S \ +; RUN: -passes='cgscc(repeat<2>(inline,function(coro-elide,dce)))' \ +; RUN: | FileCheck %s declare void @print(i32) nounwind diff --git a/llvm/test/Transforms/Coroutines/coro-heap-elide.ll b/llvm/test/Transforms/Coroutines/coro-heap-elide.ll --- a/llvm/test/Transforms/Coroutines/coro-heap-elide.ll +++ b/llvm/test/Transforms/Coroutines/coro-heap-elide.ll @@ -2,6 +2,9 @@ ; elided and any tail calls referencing the coroutine frame has the tail ; call attribute removed. ; RUN: opt < %s -S -inline -coro-elide -instsimplify -simplifycfg | FileCheck %s +; RUN: opt < %s -S \ +; RUN: -passes='cgscc(inline,function(coro-elide,instsimplify,simplify-cfg))' \ +; RUN: -aa-pipeline='basic-aa' | FileCheck %s declare void @print(i32) nounwind