diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def --- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def +++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def @@ -201,4 +201,5 @@ DUMMY_MACHINE_FUNCTION_PASS("reset-machine-function", ResetMachineFunctionPass, ()) DUMMY_MACHINE_FUNCTION_PASS("machineverifier", MachineVerifierPass, ()) DUMMY_MACHINE_FUNCTION_PASS("print-machine-cycles", MachineCycleInfoPrinterPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("spill2reg", Spill2RegPass, ()) #undef DUMMY_MACHINE_FUNCTION_PASS diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -565,6 +565,9 @@ /// This pass converts conditional moves to conditional jumps when profitable. FunctionPass *createSelectOptimizePass(); + + /// This pass replaces spills to stack with spills to registers. + extern char &Spill2RegID; } // End llvm namespace #endif diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -442,6 +442,7 @@ void initializeWriteBitcodePassPass(PassRegistry&); void initializeWriteThinLTOBitcodePass(PassRegistry&); void initializeXRayInstrumentationPass(PassRegistry&); +void initializeSpill2RegPass(PassRegistry &); } // end namespace llvm diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt --- a/llvm/lib/CodeGen/CMakeLists.txt +++ b/llvm/lib/CodeGen/CMakeLists.txt @@ -203,6 +203,7 @@ SjLjEHPrepare.cpp SlotIndexes.cpp SpillPlacement.cpp + Spill2Reg.cpp SplitKit.cpp StackColoring.cpp StackMapLivenessAnalysis.cpp diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp --- a/llvm/lib/CodeGen/CodeGen.cpp +++ b/llvm/lib/CodeGen/CodeGen.cpp @@ -130,6 +130,7 @@ initializeWasmEHPreparePass(Registry); initializeWinEHPreparePass(Registry); initializeXRayInstrumentationPass(Registry); + initializeSpill2RegPass(Registry); } void LLVMInitializeCodeGen(LLVMPassRegistryRef R) { diff --git a/llvm/lib/CodeGen/Spill2Reg.cpp b/llvm/lib/CodeGen/Spill2Reg.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/CodeGen/Spill2Reg.cpp @@ -0,0 +1,56 @@ +//===- Spill2Reg.cpp - Spill To Register Optimization ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// +/// \file This file implements Spill2Reg, an optimization which selectively +/// replaces spills/reloads to/from the stack with register copies to/from the +/// vector register file. This works even on targets where load/stores have +/// similar latency to register copies because it can free up memory units which +/// helps avoid back-end stalls. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/InitializePasses.h" +#include "llvm/Support/CommandLine.h" + +using namespace llvm; + +namespace { + +class Spill2Reg : public MachineFunctionPass { +public: + static char ID; + Spill2Reg() : MachineFunctionPass(ID) { + initializeSpill2RegPass(*PassRegistry::getPassRegistry()); + } + void getAnalysisUsage(AnalysisUsage &AU) const override; + void releaseMemory() override; + bool runOnMachineFunction(MachineFunction &) override; +}; + +} // namespace + +void Spill2Reg::getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesCFG(); + MachineFunctionPass::getAnalysisUsage(AU); +} + +void Spill2Reg::releaseMemory() {} + +bool Spill2Reg::runOnMachineFunction(MachineFunction &MFn) { + llvm_unreachable("Unimplemented"); +} + +char Spill2Reg::ID = 0; + +char &llvm::Spill2RegID = Spill2Reg::ID; + +INITIALIZE_PASS_BEGIN(Spill2Reg, "spill2reg", "Spill2Reg", false, false) +INITIALIZE_PASS_END(Spill2Reg, "spill2reg", "Spill2Reg", false, false) diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -220,6 +220,11 @@ clEnumValN(CFLAAType::Both, "both", "Enable both variants of CFL-AA"))); +// Enable the Spill2Reg pass. +static cl::opt EnableSpill2Reg("enable-spill2reg", cl::Hidden, + cl::init(false), + cl::desc("Enable Spill2Reg pass")); + /// Option names for limiting the codegen pipeline. /// Those are used in error reporting and we didn't want /// to duplicate their names all over the place. @@ -1434,6 +1439,10 @@ // Finally rewrite virtual registers. addPass(&VirtRegRewriterID); + // Replace spills to stack with spills to registers. + if (EnableSpill2Reg) + addPass(&Spill2RegID); + // Regalloc scoring for ML-driven eviction - noop except when learning a new // eviction policy. addPass(createRegAllocScoringPass());