diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -92,6 +92,12 @@ /// is that of the flag: `-forget-scev-loop-unroll`. bool ForgetAllSCEVInLoopUnroll; + /// Tuning option to enable/disable coroutine intrinsic lowering. Its default + /// value is false. Frontends such as Clang may enable this conditionally. For + /// example, Clang enables this option if the flags `-std=c++2a` or above, or + /// `-fcoroutines-ts`, have been specified. + bool Coroutines; + /// Tuning option to cap the number of calls to retrive clobbering accesses in /// MemorySSA, in LICM. unsigned LicmMssaOptCap; diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -242,6 +242,7 @@ SLPVectorization = RunSLPVectorization; LoopUnrolling = true; ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll; + Coroutines = false; LicmMssaOptCap = SetLicmMssaOptCap; LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap; } @@ -721,6 +722,8 @@ EarlyFPM.addPass(SROA()); EarlyFPM.addPass(EarlyCSEPass()); EarlyFPM.addPass(LowerExpectIntrinsicPass()); + if (PTO.Coroutines) + EarlyFPM.addPass(CoroEarlyPass()); if (Level == OptimizationLevel::O3) EarlyFPM.addPass(CallSiteSplittingPass()); @@ -844,6 +847,11 @@ MainCGPipeline.addPass(AttributorCGSCCPass()); + if (PTO.Coroutines) { + MainCGPipeline.addPass(CoroSplitPass()); + MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(CoroElidePass())); + } + // Now deduce any function attributes based in the current code. MainCGPipeline.addPass(PostOrderFunctionAttrsPass()); @@ -1046,6 +1054,9 @@ // inserting redundancies into the program. This even includes SimplifyCFG. OptimizePM.addPass(SpeculateAroundPHIsPass()); + if (PTO.Coroutines) + OptimizePM.addPass(CoroCleanupPass()); + for (auto &C : OptimizerLastEPCallbacks) C(OptimizePM, Level); @@ -1129,6 +1140,12 @@ // Reduce the size of the IR as much as possible. MPM.addPass(GlobalOptPass()); + // Module simplification splits coroutines, but does not fully clean up + // coroutine intrinsics. To ensure ThinLTO optimization passes don't trip up + // on these, we schedule the cleanup here. + if (PTO.Coroutines) + MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass())); + return MPM; } @@ -1965,6 +1982,20 @@ /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr), /* IsCS */ false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile); + + // For IR that makes use of coroutines intrinsics, coroutine passes must + // be run, even at -O0. + if (PTO.Coroutines) { + MPM.addPass(createModuleToFunctionPassAdaptor(CoroEarlyPass())); + + CGSCCPassManager CGPM(DebugLogging); + CGPM.addPass(CoroSplitPass()); + CGPM.addPass(createCGSCCToFunctionPassAdaptor(CoroElidePass())); + MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); + + MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass())); + } + // Do nothing else at all! return Error::success(); } diff --git a/llvm/test/Transforms/Coroutines/ArgAddr.ll b/llvm/test/Transforms/Coroutines/ArgAddr.ll --- a/llvm/test/Transforms/Coroutines/ArgAddr.ll +++ b/llvm/test/Transforms/Coroutines/ArgAddr.ll @@ -1,6 +1,7 @@ ; Need to move users of allocas that were moved into the coroutine frame after ; coro.begin. ; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s +; RUN: opt < %s -aa-pipeline=basic-aa -passes='default' -enable-coroutines -S | FileCheck %s define nonnull i8* @f(i32 %n) { entry: diff --git a/llvm/test/Transforms/Coroutines/coro-cleanup.ll b/llvm/test/Transforms/Coroutines/coro-cleanup.ll --- a/llvm/test/Transforms/Coroutines/coro-cleanup.ll +++ b/llvm/test/Transforms/Coroutines/coro-cleanup.ll @@ -1,5 +1,6 @@ ; Make sure that all library helper coro intrinsics are lowered. ; RUN: opt < %s -O0 -enable-coroutines -S | FileCheck %s +; RUN: opt < %s -passes='default' -enable-coroutines -S | FileCheck %s ; CHECK-LABEL: @uses_library_support_coro_intrinsics( ; CHECK-NOT: @llvm.coro diff --git a/llvm/test/Transforms/Coroutines/coro-split-01.ll b/llvm/test/Transforms/Coroutines/coro-split-01.ll --- a/llvm/test/Transforms/Coroutines/coro-split-01.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-01.ll @@ -1,5 +1,6 @@ ; Tests that a coroutine is split, inlined into the caller and devirtualized. ; RUN: opt < %s -S -enable-coroutines -O2 | FileCheck %s +; RUN: opt < %s -S -enable-coroutines -passes='default' | FileCheck %s define i8* @f() { entry: diff --git a/llvm/test/Transforms/Coroutines/ex0.ll b/llvm/test/Transforms/Coroutines/ex0.ll --- a/llvm/test/Transforms/Coroutines/ex0.ll +++ b/llvm/test/Transforms/Coroutines/ex0.ll @@ -1,5 +1,6 @@ ; First example from Doc/Coroutines.rst (two block loop) ; RUN: opt < %s -enable-coroutines -O2 -S | FileCheck %s +; RUN: opt < %s -enable-coroutines -aa-pipeline=basic-aa -passes='default' -S | FileCheck %s define i8* @f(i32 %n) { entry: diff --git a/llvm/test/Transforms/Coroutines/ex1.ll b/llvm/test/Transforms/Coroutines/ex1.ll --- a/llvm/test/Transforms/Coroutines/ex1.ll +++ b/llvm/test/Transforms/Coroutines/ex1.ll @@ -1,5 +1,6 @@ ; First example from Doc/Coroutines.rst (one block loop) ; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s +; RUN: opt < %s -aa-pipeline=basic-aa -passes='default' -enable-coroutines -S | FileCheck %s define i8* @f(i32 %n) { entry: diff --git a/llvm/test/Transforms/Coroutines/ex2.ll b/llvm/test/Transforms/Coroutines/ex2.ll --- a/llvm/test/Transforms/Coroutines/ex2.ll +++ b/llvm/test/Transforms/Coroutines/ex2.ll @@ -1,5 +1,6 @@ ; Second example from Doc/Coroutines.rst (custom alloc and free functions) ; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s +; RUN: opt < %s -passes='default' -enable-coroutines -S | FileCheck %s define i8* @f(i32 %n) { entry: diff --git a/llvm/test/Transforms/Coroutines/ex3.ll b/llvm/test/Transforms/Coroutines/ex3.ll --- a/llvm/test/Transforms/Coroutines/ex3.ll +++ b/llvm/test/Transforms/Coroutines/ex3.ll @@ -1,5 +1,6 @@ ; Third example from Doc/Coroutines.rst (two suspend points) ; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s +; RUN: opt < %s -aa-pipeline=basic-aa -passes='default' -enable-coroutines -S | FileCheck %s define i8* @f(i32 %n) { entry: diff --git a/llvm/test/Transforms/Coroutines/ex4.ll b/llvm/test/Transforms/Coroutines/ex4.ll --- a/llvm/test/Transforms/Coroutines/ex4.ll +++ b/llvm/test/Transforms/Coroutines/ex4.ll @@ -1,5 +1,6 @@ ; Fourth example from Doc/Coroutines.rst (coroutine promise) ; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s +; RUN: opt < %s -passes='default' -enable-coroutines -S | FileCheck %s define i8* @f(i32 %n) { entry: diff --git a/llvm/test/Transforms/Coroutines/ex5.ll b/llvm/test/Transforms/Coroutines/ex5.ll --- a/llvm/test/Transforms/Coroutines/ex5.ll +++ b/llvm/test/Transforms/Coroutines/ex5.ll @@ -1,5 +1,6 @@ ; Fifth example from Doc/Coroutines.rst (final suspend) ; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s +; RUN: opt < %s -aa-pipeline=basic-aa -passes='default' -enable-coroutines -S | FileCheck %s define i8* @f(i32 %n) { entry: diff --git a/llvm/test/Transforms/Coroutines/phi-coro-end.ll b/llvm/test/Transforms/Coroutines/phi-coro-end.ll --- a/llvm/test/Transforms/Coroutines/phi-coro-end.ll +++ b/llvm/test/Transforms/Coroutines/phi-coro-end.ll @@ -1,5 +1,6 @@ ; Verify that we correctly handle suspend when the coro.end block contains phi ; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s +; RUN: opt < %s -aa-pipeline=basic-aa -passes='default' -enable-coroutines -S | FileCheck %s define i8* @f(i32 %n) { entry: diff --git a/llvm/test/Transforms/Coroutines/restart-trigger.ll b/llvm/test/Transforms/Coroutines/restart-trigger.ll --- a/llvm/test/Transforms/Coroutines/restart-trigger.ll +++ b/llvm/test/Transforms/Coroutines/restart-trigger.ll @@ -4,6 +4,10 @@ ; REQUIRES: asserts ; RUN: opt < %s -S -O0 -enable-coroutines -debug-only=coro-split 2>&1 | FileCheck %s ; RUN: opt < %s -S -O1 -enable-coroutines -debug-only=coro-split 2>&1 | FileCheck %s +; The following tests use the new pass manager, and verify that the coroutine +; passes re-run the CGSCC pipeline. +; RUN: opt < %s -S -passes='default' -enable-coroutines -debug-only=coro-split 2>&1 | FileCheck %s +; RUN: opt < %s -S -passes='default' -enable-coroutines -debug-only=coro-split 2>&1 | FileCheck %s ; CHECK: CoroSplit: Processing coroutine 'f' state: 0 ; CHECK-NEXT: CoroSplit: Processing coroutine 'f' state: 1 diff --git a/llvm/test/Transforms/Coroutines/smoketest.ll b/llvm/test/Transforms/Coroutines/smoketest.ll --- a/llvm/test/Transforms/Coroutines/smoketest.ll +++ b/llvm/test/Transforms/Coroutines/smoketest.ll @@ -1,6 +1,7 @@ ; Test that all coroutine passes run in the correct order at all optimization ; levels and -enable-coroutines adds coroutine passes to the pipeline. ; +; Legacy pass manager: ; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments -O0 2>&1 | FileCheck %s ; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments -O1 2>&1 | FileCheck %s ; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments -O2 2>&1 | FileCheck %s @@ -9,6 +10,18 @@ ; RUN: -coro-early -coro-split -coro-elide -coro-cleanup 2>&1 | FileCheck %s ; RUN: opt < %s -disable-output -debug-pass=Arguments 2>&1 \ ; RUN: | FileCheck %s -check-prefix=NOCORO +; New pass manager: +; RUN: opt < %s -disable-output -passes='default' -enable-coroutines \ +; RUN: -debug-pass-manager 2>&1 | FileCheck %s -check-prefix=NEWPM +; RUN: opt < %s -disable-output -passes='default' -enable-coroutines \ +; RUN: -debug-pass-manager 2>&1 | FileCheck %s -check-prefix=NEWPM +; RUN: opt < %s -disable-output -passes='default' -enable-coroutines \ +; RUN: -debug-pass-manager 2>&1 | FileCheck %s -check-prefix=NEWPM +; RUN: opt < %s -disable-output -passes='default' -enable-coroutines \ +; RUN: -debug-pass-manager 2>&1 | FileCheck %s -check-prefix=NEWPM +; RUN: opt < %s -disable-output -debug-pass-manager \ +; RUN: -passes='function(coro-early),cgscc(coro-split),function(coro-elide,coro-cleanup)' 2>&1 \ +; RUN: | FileCheck %s -check-prefix=NEWPM ; CHECK: coro-early ; CHECK: coro-split @@ -20,6 +33,11 @@ ; NOCORO-NOT: coro-elide ; NOCORO-NOT: coro-cleanup +; NEWPM: CoroEarlyPass +; NEWPM: CoroSplitPass +; NEWPM: CoroElidePass +; NEWPM: CoroCleanupPass + define void @foo() { ret void } diff --git a/llvm/tools/opt/NewPMDriver.h b/llvm/tools/opt/NewPMDriver.h --- a/llvm/tools/opt/NewPMDriver.h +++ b/llvm/tools/opt/NewPMDriver.h @@ -64,7 +64,7 @@ bool ShouldPreserveAssemblyUseListOrder, bool ShouldPreserveBitcodeUseListOrder, bool EmitSummaryIndex, bool EmitModuleHash, - bool EnableDebugify); + bool EnableDebugify, bool Coroutines); } // namespace llvm #endif diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp --- a/llvm/tools/opt/NewPMDriver.cpp +++ b/llvm/tools/opt/NewPMDriver.cpp @@ -214,7 +214,7 @@ bool ShouldPreserveAssemblyUseListOrder, bool ShouldPreserveBitcodeUseListOrder, bool EmitSummaryIndex, bool EmitModuleHash, - bool EnableDebugify) { + bool EnableDebugify, bool Coroutines) { bool VerifyEachPass = VK == VK_VerifyEachPass; Optional P; @@ -259,7 +259,9 @@ StandardInstrumentations SI; SI.registerCallbacks(PIC); - PassBuilder PB(TM, PipelineTuningOptions(), P, &PIC); + PipelineTuningOptions PTO; + PTO.Coroutines = Coroutines; + PassBuilder PB(TM, PTO, P, &PIC); registerEPCallbacks(PB, VerifyEachPass, DebugPM); // Load requested pass plugins and let them register pass builder callbacks diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp --- a/llvm/tools/opt/opt.cpp +++ b/llvm/tools/opt/opt.cpp @@ -716,7 +716,7 @@ RemarksFile.get(), PassPipeline, OK, VK, PreserveAssemblyUseListOrder, PreserveBitcodeUseListOrder, EmitSummaryIndex, - EmitModuleHash, EnableDebugify) + EmitModuleHash, EnableDebugify, Coroutines) ? 0 : 1; }