diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -49,6 +49,10 @@ #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Transforms/Coroutines.h" +#include "llvm/Transforms/Coroutines/CoroCleanup.h" +#include "llvm/Transforms/Coroutines/CoroEarly.h" +#include "llvm/Transforms/Coroutines/CoroElide.h" +#include "llvm/Transforms/Coroutines/CoroSplit.h" #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/LowerTypeTests.h" @@ -957,6 +961,22 @@ } } +static void addCoroutinePassesAtO0(ModulePassManager &MPM, + const LangOptions &LangOpts, + const CodeGenOptions &CodeGenOpts) { + if (!LangOpts.Coroutines) + return; + + MPM.addPass(createModuleToFunctionPassAdaptor(CoroEarlyPass())); + + CGSCCPassManager CGPM(CodeGenOpts.DebugPassManager); + CGPM.addPass(CoroSplitPass()); + CGPM.addPass(createCGSCCToFunctionPassAdaptor(CoroElidePass())); + MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); + + MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass())); +} + static void addSanitizersAtO0(ModulePassManager &MPM, const Triple &TargetTriple, const LangOptions &LangOpts, @@ -1076,6 +1096,7 @@ PTO.LoopInterleaving = CodeGenOpts.UnrollLoops; PTO.LoopVectorization = CodeGenOpts.VectorizeLoop; PTO.SLPVectorization = CodeGenOpts.VectorizeSLP; + PTO.Coroutines = LangOpts.Coroutines; PassInstrumentationCallbacks PIC; StandardInstrumentations SI; @@ -1279,6 +1300,7 @@ } if (CodeGenOpts.OptimizationLevel == 0) { + addCoroutinePassesAtO0(MPM, LangOpts, CodeGenOpts); addSanitizersAtO0(MPM, TargetTriple, LangOpts, CodeGenOpts); } } diff --git a/clang/test/CodeGenCoroutines/coro-newpm-pipeline.cpp b/clang/test/CodeGenCoroutines/coro-newpm-pipeline.cpp new file mode 100644 --- /dev/null +++ b/clang/test/CodeGenCoroutines/coro-newpm-pipeline.cpp @@ -0,0 +1,57 @@ +// Tests that coroutine passes are added to and run by the new pass manager +// pipeline, at -O0 and above. + +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm-bc -o /dev/null \ +// RUN: -fexperimental-new-pass-manager -fdebug-pass-manager -fcoroutines-ts \ +// RUN: -O0 %s 2>&1 | FileCheck %s +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm-bc -o /dev/null \ +// RUN: -fexperimental-new-pass-manager -fdebug-pass-manager -fcoroutines-ts \ +// RUN: -O1 %s 2>&1 | FileCheck %s +// +// CHECK: Starting llvm::Module pass manager run. +// CHECK: Running pass:{{.*}}CoroEarlyPass +// +// The first coro-split pass enqueues a second run of the entire CGSCC pipeline. +// CHECK: Starting CGSCC pass manager run. +// CHECK: Running pass: CoroSplitPass on (_Z3foov) +// CHECK: Running pass:{{.*}}CoroElidePass{{.*}} on (_Z3foov) +// CHECK: Finished CGSCC pass manager run. +// +// The second coro-split pass splits coroutine 'foo' into funclets +// 'foo.resume', 'foo.destroy', and 'foo.cleanup'. +// CHECK: Starting CGSCC pass manager run. +// CHECK: Running pass: CoroSplitPass on (_Z3foov) +// CHECK: Running pass:{{.*}}CoroElidePass{{.*}} on (_Z3foov) +// CHECK: Finished CGSCC pass manager run. +// +// CHECK: Running pass:{{.*}}CoroCleanupPass +// CHECK: Finished llvm::Module pass manager run. + +namespace std { +namespace experimental { + +struct handle {}; + +struct awaitable { + bool await_ready() { return true; } + void await_suspend(handle) {} + bool await_resume() { return true; } +}; + +template struct coroutine_handle { + static handle from_address(void *address) { return {}; } +}; + +template struct coroutine_traits { + struct promise_type { + awaitable initial_suspend() { return {}; } + awaitable final_suspend() { return {}; } + void return_void() {} + T get_return_object() { return T(); } + void unhandled_exception() {} + }; +}; +} // namespace experimental +} // namespace std + +void foo() { co_return; } diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -92,6 +92,12 @@ /// is that of the flag: `-forget-scev-loop-unroll`. bool ForgetAllSCEVInLoopUnroll; + /// Tuning option to enable/disable coroutine intrinsic lowering. Its default + /// value is false. Frontends such as Clang may enable this conditionally. For + /// example, Clang enables this option if the flags `-std=c++2a` or above, or + /// `-fcoroutines-ts`, have been specified. + bool Coroutines; + /// Tuning option to cap the number of calls to retrive clobbering accesses in /// MemorySSA, in LICM. unsigned LicmMssaOptCap; diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -242,6 +242,7 @@ SLPVectorization = RunSLPVectorization; LoopUnrolling = true; ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll; + Coroutines = false; LicmMssaOptCap = SetLicmMssaOptCap; LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap; } @@ -721,6 +722,8 @@ EarlyFPM.addPass(SROA()); EarlyFPM.addPass(EarlyCSEPass()); EarlyFPM.addPass(LowerExpectIntrinsicPass()); + if (PTO.Coroutines) + EarlyFPM.addPass(CoroEarlyPass()); if (Level == OptimizationLevel::O3) EarlyFPM.addPass(CallSiteSplittingPass()); @@ -844,6 +847,11 @@ MainCGPipeline.addPass(AttributorCGSCCPass()); + if (PTO.Coroutines) { + MainCGPipeline.addPass(CoroSplitPass()); + MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(CoroElidePass())); + } + // Now deduce any function attributes based in the current code. MainCGPipeline.addPass(PostOrderFunctionAttrsPass()); @@ -1046,6 +1054,9 @@ // inserting redundancies into the program. This even includes SimplifyCFG. OptimizePM.addPass(SpeculateAroundPHIsPass()); + if (PTO.Coroutines) + OptimizePM.addPass(CoroCleanupPass()); + for (auto &C : OptimizerLastEPCallbacks) C(OptimizePM, Level); @@ -1129,6 +1140,12 @@ // Reduce the size of the IR as much as possible. MPM.addPass(GlobalOptPass()); + // Module simplification splits coroutines, but does not fully clean up + // coroutine intrinsics. To ensure ThinLTO optimization passes don't trip up + // on these, we schedule the cleanup here. + if (PTO.Coroutines) + MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass())); + return MPM; }