Index: docs/Coroutines.rst =================================================================== --- docs/Coroutines.rst +++ docs/Coroutines.rst @@ -1177,8 +1177,8 @@ Upstreaming sequence (rough plan) ================================= #. Add documentation. -#. Add coroutine intrinsics. <= we are here -#. Add empty coroutine passes. +#. Add coroutine intrinsics. +#. Add empty coroutine passes. <= we are here #. Add coroutine devirtualization + tests. #. Add CGSCC restart trigger + tests. #. Add coroutine heap elision + tests. Index: include/llvm/InitializePasses.h =================================================================== --- include/llvm/InitializePasses.h +++ include/llvm/InitializePasses.h @@ -46,6 +46,9 @@ /// Initialize all passes linked into the Analysis library. void initializeAnalysis(PassRegistry&); +/// Initialize all passes linked into the Coroutines library. +void initializeCoroutines(PassRegistry&); + /// Initialize all passes linked into the CodeGen library. void initializeCodeGen(PassRegistry&); Index: include/llvm/Transforms/Coroutines.h =================================================================== --- /dev/null +++ include/llvm/Transforms/Coroutines.h @@ -0,0 +1,38 @@ +//===-- Coroutines.h - Coroutine Transformations ----------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Declare accessor functions for coroutine lowering passes. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_COROUTINES_H +#define LLVM_TRANSFORMS_COROUTINES_H + +namespace llvm { + +class Pass; +class PassManagerBuilder; + +/// Add all coroutine passes to appropriate extension points. +void addCoroutinePassesToExtensionPoints(PassManagerBuilder &Builder); + +/// Lower coroutine intrinsics that are not needed by later passes. +Pass *createCoroEarlyPass(); + +/// Split up coroutines into multiple functions driving their state machines. +Pass *createCoroSplitPass(); + +/// Analyze coroutines use sites, devirtualize resume/destroy calls and elide +/// heap allocation for coroutine frame where possible. +Pass *createCoroElidePass(); + +/// Lower all remaining coroutine intrinsics. +Pass *createCoroCleanupPass(); + +} + +#endif Index: include/llvm/Transforms/IPO/PassManagerBuilder.h =================================================================== --- include/llvm/Transforms/IPO/PassManagerBuilder.h +++ include/llvm/Transforms/IPO/PassManagerBuilder.h @@ -100,6 +100,11 @@ /// peephole optimizations similar to the instruction combiner. These passes /// will be inserted after each instance of the instruction combiner pass. EP_Peephole, + + /// EP_CGSCCOptimizerLate - This extension point allows adding CallGraphSCC + /// passes at the end of the main CallGraphSCC passes and before any + /// function simplification passes run by CGPassManager. + EP_CGSCCOptimizerLate, }; /// The Optimization Level - Specify the basic optimization level. Index: lib/Transforms/CMakeLists.txt =================================================================== --- lib/Transforms/CMakeLists.txt +++ lib/Transforms/CMakeLists.txt @@ -6,3 +6,4 @@ add_subdirectory(Vectorize) add_subdirectory(Hello) add_subdirectory(ObjCARC) +add_subdirectory(Coroutines) Index: lib/Transforms/Coroutines/CMakeLists.txt =================================================================== --- /dev/null +++ lib/Transforms/Coroutines/CMakeLists.txt @@ -0,0 +1,8 @@ +add_llvm_library(LLVMCoroutines + Coroutines.cpp + CoroCleanup.cpp + CoroEarly.cpp + CoroElide.cpp + CoroSplit.cpp + ) + Index: lib/Transforms/Coroutines/CoroCleanup.cpp =================================================================== --- /dev/null +++ lib/Transforms/Coroutines/CoroCleanup.cpp @@ -0,0 +1,42 @@ +//===- CoroCleanup.cpp - Coroutine Cleanup Pass ---------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// This pass lowers all remaining coroutine intrinsics. +//===----------------------------------------------------------------------===// + +#include "CoroInternal.h" +#include "llvm/Pass.h" + +using namespace llvm; + +#define DEBUG_TYPE "coro-cleanup" + +//===----------------------------------------------------------------------===// +// Top Level Driver +//===----------------------------------------------------------------------===// + +namespace { + +struct CoroCleanup : FunctionPass { + static char ID; // Pass identification, replacement for typeid + + CoroCleanup() : FunctionPass(ID) {} + + bool runOnFunction(Function &F) override { return false; } + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesAll(); + } +}; + +} + +char CoroCleanup::ID = 0; +INITIALIZE_PASS(CoroCleanup, "coro-cleanup", + "Lower all coroutine related intrinsics", false, false) + +Pass *llvm::createCoroCleanupPass() { return new CoroCleanup(); } Index: lib/Transforms/Coroutines/CoroEarly.cpp =================================================================== --- /dev/null +++ lib/Transforms/Coroutines/CoroEarly.cpp @@ -0,0 +1,43 @@ +//===- CoroEarly.cpp - Coroutine Early Function Pass ----------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// This pass lowers coroutine intrinsics that hide the details of the exact +// calling convention for coroutine resume and destroy functions and details of +// the structure of the coroutine frame. +//===----------------------------------------------------------------------===// + +#include "CoroInternal.h" +#include "llvm/Pass.h" + +using namespace llvm; + +#define DEBUG_TYPE "coro-early" + +//===----------------------------------------------------------------------===// +// Top Level Driver +//===----------------------------------------------------------------------===// + +namespace { + +struct CoroEarly : public FunctionPass { + static char ID; // Pass identification, replacement for typeid + CoroEarly() : FunctionPass(ID) {} + + bool runOnFunction(Function &F) override { return false; } + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesAll(); + } +}; + +} + +char CoroEarly::ID = 0; +INITIALIZE_PASS(CoroEarly, "coro-early", "Lower early coroutine intrinsics", + false, false) + +Pass *llvm::createCoroEarlyPass() { return new CoroEarly(); } Index: lib/Transforms/Coroutines/CoroElide.cpp =================================================================== --- /dev/null +++ lib/Transforms/Coroutines/CoroElide.cpp @@ -0,0 +1,49 @@ +//===- CoroElide.cpp - Coroutine Frame Allocation Elision Pass ------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// This pass replaces dynamic allocation of coroutine frame with alloca and +// replaces calls to llvm.coro.resume and llvm.coro.destroy with direct calls +// to coroutine sub-functions. +//===----------------------------------------------------------------------===// + +#include "CoroInternal.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Pass.h" + +using namespace llvm; + +#define DEBUG_TYPE "coro-elide" + +//===----------------------------------------------------------------------===// +// Top Level Driver +//===----------------------------------------------------------------------===// + +namespace { + +struct CoroElide : FunctionPass { + static char ID; + CoroElide() : FunctionPass(ID) {} + bool runOnFunction(Function &F) override { return false; } + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesAll(); + } +}; + +} + +char CoroElide::ID = 0; +INITIALIZE_PASS_BEGIN( + CoroElide, "coro-elide", + "Coroutine frame allocation elision and indirect calls replacement", false, + false) +INITIALIZE_PASS_END( + CoroElide, "coro-elide", + "Coroutine frame allocation elision and indirect calls replacement", false, + false) + +Pass *llvm::createCoroElidePass() { return new CoroElide(); } Index: lib/Transforms/Coroutines/CoroInternal.h =================================================================== --- /dev/null +++ lib/Transforms/Coroutines/CoroInternal.h @@ -0,0 +1,28 @@ +//===- CoroInternal.h - Internal Coroutine interfaces ---------*- C++ -*---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Common definitions/declarations used internally by coroutine lowering passes. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TRANSFORMS_COROUTINES_COROINTERNAL_H +#define LLVM_LIB_TRANSFORMS_COROUTINES_COROINTERNAL_H + +#include "llvm/Transforms/Coroutines.h" + +namespace llvm { + +class PassRegistry; + +void initializeCoroEarlyPass(PassRegistry &); +void initializeCoroSplitPass(PassRegistry &); +void initializeCoroElidePass(PassRegistry &); +void initializeCoroCleanupPass(PassRegistry &); + +} + +#endif Index: lib/Transforms/Coroutines/CoroSplit.cpp =================================================================== --- /dev/null +++ lib/Transforms/Coroutines/CoroSplit.cpp @@ -0,0 +1,45 @@ +//===- CoroSplit.cpp - Converts a coroutine into a state machine ----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// This pass builds the coroutine frame and outlines resume and destroy parts +// of the coroutine into separate functions. +//===----------------------------------------------------------------------===// + +#include "CoroInternal.h" +#include "llvm/Analysis/CallGraphSCCPass.h" + +using namespace llvm; + +#define DEBUG_TYPE "coro-split" + +//===----------------------------------------------------------------------===// +// Top Level Driver +//===----------------------------------------------------------------------===// + +namespace { + +struct CoroSplit : public CallGraphSCCPass { + static char ID; // Pass identification, replacement for typeid + CoroSplit() : CallGraphSCCPass(ID) {} + + bool runOnSCC(CallGraphSCC &SCC) override { return false; } + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesAll(); + CallGraphSCCPass::getAnalysisUsage(AU); + } +}; + +} + +char CoroSplit::ID = 0; +INITIALIZE_PASS( + CoroSplit, "coro-split", + "Split coroutine into a set of functions driving its state machine", false, + false) + +Pass *llvm::createCoroSplitPass() { return new CoroSplit(); } Index: lib/Transforms/Coroutines/Coroutines.cpp =================================================================== --- /dev/null +++ lib/Transforms/Coroutines/Coroutines.cpp @@ -0,0 +1,68 @@ +//===-- Coroutines.cpp ----------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// This file implements the common infrastructure for Coroutine Passes. +//===----------------------------------------------------------------------===// + +#include "CoroInternal.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Verifier.h" +#include "llvm/InitializePasses.h" +#include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" + +using namespace llvm; + +void llvm::initializeCoroutines(PassRegistry &Registry) { + initializeCoroEarlyPass(Registry); + initializeCoroSplitPass(Registry); + initializeCoroElidePass(Registry); + initializeCoroCleanupPass(Registry); +} + +static void addCoroutineOpt0Passes(const PassManagerBuilder &Builder, + legacy::PassManagerBase &PM) { + PM.add(createCoroSplitPass()); + PM.add(createCoroElidePass()); + + PM.add(createBarrierNoopPass()); + PM.add(createCoroCleanupPass()); +} + +static void addCoroutineEarlyPasses(const PassManagerBuilder &Builder, + legacy::PassManagerBase &PM) { + PM.add(createCoroEarlyPass()); +} + +static void addCoroutineScalarOptimizerPasses(const PassManagerBuilder &Builder, + legacy::PassManagerBase &PM) { + PM.add(createCoroElidePass()); +} + +static void addCoroutineSCCPasses(const PassManagerBuilder &Builder, + legacy::PassManagerBase &PM) { + PM.add(createCoroSplitPass()); +} + +static void addCoroutineOptimizerLastPasses(const PassManagerBuilder &Builder, + legacy::PassManagerBase &PM) { + PM.add(createCoroCleanupPass()); +} + +void llvm::addCoroutinePassesToExtensionPoints(PassManagerBuilder &Builder) { + Builder.addExtension(PassManagerBuilder::EP_EarlyAsPossible, + addCoroutineEarlyPasses); + Builder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, + addCoroutineOpt0Passes); + Builder.addExtension(PassManagerBuilder::EP_CGSCCOptimizerLate, + addCoroutineSCCPasses); + Builder.addExtension(PassManagerBuilder::EP_ScalarOptimizerLate, + addCoroutineScalarOptimizerPasses); + Builder.addExtension(PassManagerBuilder::EP_OptimizerLast, + addCoroutineOptimizerLastPasses); +} Index: lib/Transforms/Coroutines/LLVMBuild.txt =================================================================== --- /dev/null +++ lib/Transforms/Coroutines/LLVMBuild.txt @@ -0,0 +1,22 @@ +;===- ./lib/Transforms/Coroutines/LLVMBuild.txt ----------------*- Conf -*--===; +; +; The LLVM Compiler Infrastructure +; +; This file is distributed under the University of Illinois Open Source +; License. See LICENSE.TXT for details. +; +;===------------------------------------------------------------------------===; +; +; This is an LLVMBuild description file for the components in this subdirectory. +; +; For more information on the LLVMBuild system, please see: +; +; http://llvm.org/docs/LLVMBuild.html +; +;===------------------------------------------------------------------------===; + +[component_0] +type = Library +name = Coroutines +parent = Transforms +required_libraries = Analysis Core IPO Scalar Support TransformUtils Index: lib/Transforms/IPO/PassManagerBuilder.cpp =================================================================== --- lib/Transforms/IPO/PassManagerBuilder.cpp +++ lib/Transforms/IPO/PassManagerBuilder.cpp @@ -440,6 +440,7 @@ if (OptLevel > 2) MPM.add(createArgumentPromotionPass()); // Scalarize uninlined fn args + addExtensionsToPM(EP_CGSCCOptimizerLate, MPM); addFunctionSimplificationPasses(MPM); // FIXME: This is a HACK! The inliner pass above implicitly creates a CGSCC Index: lib/Transforms/LLVMBuild.txt =================================================================== --- lib/Transforms/LLVMBuild.txt +++ lib/Transforms/LLVMBuild.txt @@ -16,7 +16,7 @@ ;===------------------------------------------------------------------------===; [common] -subdirectories = IPO InstCombine Instrumentation Scalar Utils Vectorize ObjCARC +subdirectories = Coroutines IPO InstCombine Instrumentation Scalar Utils Vectorize ObjCARC [component_0] type = Group Index: test/Transforms/Coroutines/smoketest.ll =================================================================== --- /dev/null +++ test/Transforms/Coroutines/smoketest.ll @@ -0,0 +1,24 @@ +; Test that all coroutine passes run in the correct order at all optimization +; levels and -disable-coroutines removes coroutine passes from the pipeline. +; +; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments -O1 2>&1 | FileCheck %s +; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments -O2 2>&1 | FileCheck %s +; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments -O3 2>&1 | FileCheck %s +; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments \ +; RUN: -coro-early -coro-split -coro-elide -coro-cleanup 2>&1 | FileCheck %s +; RUN: opt < %s -disable-output -debug-pass=Arguments 2>&1 \ +; RUN: | FileCheck %s -check-prefix=NOCORO + +; CHECK: coro-early +; CHECK: coro-split +; CHECK: coro-elide +; CHECK: coro-cleanup + +; NOCORO-NOT: coro-early +; NOCORO-NOT: coro-split +; NOCORO-NOT: coro-elide +; NOCORO-NOT: coro-cleanup + +define void @foo() { + ret void +} Index: tools/opt/CMakeLists.txt =================================================================== --- tools/opt/CMakeLists.txt +++ tools/opt/CMakeLists.txt @@ -4,6 +4,7 @@ BitWriter CodeGen Core + Coroutines IPO IRReader InstCombine Index: tools/opt/opt.cpp =================================================================== --- tools/opt/opt.cpp +++ tools/opt/opt.cpp @@ -50,6 +50,7 @@ #include "llvm/Support/TargetSelect.h" #include "llvm/Support/ToolOutputFile.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/Coroutines.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/Utils/Cloning.h" #include @@ -215,6 +216,11 @@ cl::desc("Discard names from Value (other than GlobalValue)."), cl::init(false), cl::Hidden); +static cl::opt Coroutines( + "enable-coroutines", + cl::desc("Enable coroutine passes."), + cl::init(false), cl::Hidden); + static cl::opt PassRemarksWithHotness( "pass-remarks-with-hotness", cl::desc("With PGO, include profile count in optimization remarks"), @@ -274,6 +280,9 @@ TM->addEarlyAsPossiblePasses(PM); }); + if (Coroutines) + addCoroutinePassesToExtensionPoints(Builder); + Builder.populateFunctionPassManager(FPM); Builder.populateModulePassManager(MPM); } @@ -348,6 +357,7 @@ // Initialize passes PassRegistry &Registry = *PassRegistry::getPassRegistry(); initializeCore(Registry); + initializeCoroutines(Registry); initializeScalarOpts(Registry); initializeObjCARCOpts(Registry); initializeVectorization(Registry);