Index: include/llvm/CodeGen/Passes.h =================================================================== --- include/llvm/CodeGen/Passes.h +++ include/llvm/CodeGen/Passes.h @@ -481,6 +481,12 @@ /// and eliminates abstract frame references. extern char &PrologEpilogCodeInserterID; + /// Prolog/Epilog inserter without CSRs - A version of PEI that does not use + /// or support callee-saved registers or scavenging, but does support use of + /// virtual registers instead of only physical registers. Targets can + /// substitute it for the default PrologEpilogCodeInserterID. + extern char &VRegPrologEpilogCodeInserterID; + /// ExpandPostRAPseudos - This pass expands pseudo instructions after /// register allocation. extern char &ExpandPostRAPseudosID; Index: include/llvm/InitializePasses.h =================================================================== --- include/llvm/InitializePasses.h +++ include/llvm/InitializePasses.h @@ -219,6 +219,7 @@ void initializeOptimizePHIsPass(PassRegistry&); void initializePartiallyInlineLibCallsPass(PassRegistry&); void initializePEIPass(PassRegistry&); +void initializePEIWithCSRsPass(PassRegistry&); void initializePHIEliminationPass(PassRegistry&); void initializePartialInlinerPass(PassRegistry&); void initializePeepholeOptimizerPass(PassRegistry&); Index: lib/CodeGen/CodeGen.cpp =================================================================== --- lib/CodeGen/CodeGen.cpp +++ lib/CodeGen/CodeGen.cpp @@ -57,6 +57,7 @@ initializeMachineVerifierPassPass(Registry); initializeOptimizePHIsPass(Registry); initializePEIPass(Registry); + initializePEIWithCSRsPass(Registry); initializePHIEliminationPass(Registry); initializePeepholeOptimizerPass(Registry); initializePostMachineSchedulerPass(Registry); Index: lib/CodeGen/PrologEpilogInserter.cpp =================================================================== --- lib/CodeGen/PrologEpilogInserter.cpp +++ lib/CodeGen/PrologEpilogInserter.cpp @@ -50,12 +50,15 @@ #define DEBUG_TYPE "pei" namespace { +// The PrologEpilogInserter base class performs frame finalization, +// prolog/epilog code insertion, and FrameIndex elimination. It does not handle +// callee-saved registers or scavenging, but it does support virtual registers +// (i.e. it is suitable for virtual targets which do not use a register +// allocator) class PEI : public MachineFunctionPass { public: static char ID; - PEI() : MachineFunctionPass(ID) { - initializePEIPass(*PassRegistry::getPassRegistry()); - } + PEI() : PEI(ID) { initializePEIPass(*PassRegistry::getPassRegistry()); } void getAnalysisUsage(AnalysisUsage &AU) const override; @@ -64,54 +67,91 @@ /// bool runOnMachineFunction(MachineFunction &Fn) override; -private: - RegScavenger *RS; + protected: + PEI(char &SubclassID) : MachineFunctionPass(SubclassID) {} + + // Register scavenging is only supported in PEIWithCSRs and not in the base + // class. However, an RS pointer (possibly null) must be passed to several + // of the target hooks used in the base class and the logic for FI + // replacement sometimes depends on it. + RegScavenger *RS = nullptr; + + // Flag to control whether to use the register scavenger to resolve + // frame index materialization registers. Set according to + // TRI->requiresFrameIndexScavenging() for the current function. + bool FrameIndexVirtualScavenging = false; // MinCSFrameIndex, MaxCSFrameIndex - Keeps the range of callee saved // stack frame indexes. - unsigned MinCSFrameIndex, MaxCSFrameIndex; + unsigned MinCSFrameIndex = std::numeric_limits::max(); + unsigned MaxCSFrameIndex = 0; // Save and Restore blocks of the current function. Typically there is a // single save block, unless Windows EH funclets are involved. SmallVector SaveBlocks; SmallVector RestoreBlocks; - // Flag to control whether to use the register scavenger to resolve - // frame index materialization registers. Set according to - // TRI->requiresFrameIndexScavenging() for the current function. - bool FrameIndexVirtualScavenging; - + void init(MachineFunction &Fn); void calculateSets(MachineFunction &Fn); void calculateCallsInformation(MachineFunction &Fn); - void assignCalleeSavedSpillSlots(MachineFunction &Fn, - const BitVector &SavedRegs); - void insertCSRSpillsAndRestores(MachineFunction &Fn); + + void finalizeFrame(MachineFunction &Fn); void calculateFrameObjectOffsets(MachineFunction &Fn); void replaceFrameIndices(MachineFunction &Fn); void replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn, int &SPAdj); - void scavengeFrameVirtualRegs(MachineFunction &Fn); + void insertPrologEpilogCode(MachineFunction &Fn); }; + +class PEIWithCSRs : public PEI { + public: + static char ID; + PEIWithCSRs() : PEI(ID) { + initializePEIWithCSRsPass(*PassRegistry::getPassRegistry()); + } + /// runOnMachineFunction - Insert prolog/epilog code and replace abstract + /// frame indexes with appropriate references. + /// + bool runOnMachineFunction(MachineFunction &Fn) override; + + private: + void assignCalleeSavedSpillSlots(MachineFunction &Fn, + const BitVector &SavedRegs); + void insertCSRSpillsAndRestores(MachineFunction &Fn); + void scavengeFrameVirtualRegs(MachineFunction &Fn); +}; } // namespace char PEI::ID = 0; -char &llvm::PrologEpilogCodeInserterID = PEI::ID; +char PEIWithCSRs::ID = 0; +char &llvm::PrologEpilogCodeInserterID = PEIWithCSRs::ID; +char &llvm::VRegPrologEpilogCodeInserterID = PEI::ID; static cl::opt WarnStackSize("warn-stack-size", cl::Hidden, cl::init((unsigned)-1), cl::desc("Warn for stack size bigger than the given" " number")); -INITIALIZE_PASS_BEGIN(PEI, "prologepilog", - "Prologue/Epilogue Insertion", false, false) +INITIALIZE_PASS_BEGIN(PEI, "prologepilog-no-csrs", + "Prologue/Epilogue Insertion (no CSRs)", false, false) INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) INITIALIZE_PASS_DEPENDENCY(StackProtector) INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) -INITIALIZE_PASS_END(PEI, "prologepilog", - "Prologue/Epilogue Insertion & Frame Finalization", - false, false) +INITIALIZE_PASS_END( + PEI, "prologepilog-no-csrs", + "Prologue/Epilogue Insertion & Frame Finalization (no CSRs)", false, false) + +INITIALIZE_PASS_BEGIN(PEIWithCSRs, "prologepilog", + "Prologue/Epilogue Insertion", false, false) +INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) +INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) +INITIALIZE_PASS_DEPENDENCY(StackProtector) +INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) +INITIALIZE_PASS_END(PEIWithCSRs, "prologepilog", + "Prologue/Epilogue Insertion & Frame Finalization", false, + false) STATISTIC(NumScavengedRegs, "Number of frame index regs scavenged"); STATISTIC(NumBytesStackSpace, @@ -160,38 +200,34 @@ /// StackObjSet - A set of stack object indexes typedef SmallSetVector StackObjSet; -/// runOnMachineFunction - Insert prolog/epilog code and replace abstract -/// frame indexes with appropriate references. -/// -bool PEI::runOnMachineFunction(MachineFunction &Fn) { - const Function* F = Fn.getFunction(); - const TargetRegisterInfo *TRI = Fn.getSubtarget().getRegisterInfo(); - const TargetFrameLowering *TFI = Fn.getSubtarget().getFrameLowering(); - - assert(!Fn.getRegInfo().getNumVirtRegs() && "Regalloc must assign all vregs"); - - RS = TRI->requiresRegisterScavenging(Fn) ? new RegScavenger() : nullptr; - FrameIndexVirtualScavenging = TRI->requiresFrameIndexScavenging(Fn); - +void PEI::init(MachineFunction &Fn) { // Calculate the MaxCallFrameSize and AdjustsStack variables for the // function's frame information. Also eliminates call frame pseudo // instructions. calculateCallsInformation(Fn); - // Determine which of the registers in the callee save list should be saved. - BitVector SavedRegs; - TFI->determineCalleeSaves(Fn, SavedRegs, RS); - - // Insert spill code for any callee saved registers that are modified. - assignCalleeSavedSpillSlots(Fn, SavedRegs); - // Determine placement of CSR spill/restore code: // place all spills in the entry block, all restores in return blocks. calculateSets(Fn); +} - // Add the code to save and restore the callee saved registers. - if (!F->hasFnAttribute(Attribute::Naked)) - insertCSRSpillsAndRestores(Fn); +/// runOnMachineFunction - Insert prolog/epilog code and replace abstract +/// frame indexes with appropriate references. +/// +bool PEI::runOnMachineFunction(MachineFunction &Fn) { + assert(!RS && "Base PEI does not support scavenging"); + + init(Fn); + finalizeFrame(Fn); + + return true; +} + +/// Finalize the frame layout, insert prolog and epilog code, and replace +/// FrameIndex operands. +void PEI::finalizeFrame(MachineFunction &Fn) { + const Function *F = Fn.getFunction(); + const TargetFrameLowering *TFI = Fn.getSubtarget().getFrameLowering(); // Allow the target machine to make final modifications to the function // before the frame layout is finalized. @@ -213,15 +249,6 @@ // replaceFrameIndices(Fn); - // If register scavenging is needed, as we've enabled doing it as a - // post-pass, scavenge the virtual registers that frame index elimination - // inserted. - if (TRI->requiresRegisterScavenging(Fn) && FrameIndexVirtualScavenging) - scavengeFrameVirtualRegs(Fn); - - // Clear any vregs created by virtual scavenging. - Fn.getRegInfo().clearVirtRegs(); - // Warn on stack size when we exceeds the given limit. MachineFrameInfo *MFI = Fn.getFrameInfo(); uint64_t StackSize = MFI->getStackSize(); @@ -230,9 +257,42 @@ F->getContext().diagnose(DiagStackSize); } - delete RS; SaveBlocks.clear(); RestoreBlocks.clear(); +} + +bool PEIWithCSRs::runOnMachineFunction(MachineFunction &Fn) { + const TargetFrameLowering *TFI = Fn.getSubtarget().getFrameLowering(); + const TargetRegisterInfo *TRI = Fn.getSubtarget().getRegisterInfo(); + RS = TRI->requiresRegisterScavenging(Fn) ? new RegScavenger() : nullptr; + FrameIndexVirtualScavenging = TRI->requiresFrameIndexScavenging(Fn); + + assert(!Fn.getRegInfo().getNumVirtRegs() && "Regalloc must assign all vregs"); + + init(Fn); + + // Determine which of the registers in the callee save list should be saved. + BitVector SavedRegs; + TFI->determineCalleeSaves(Fn, SavedRegs, RS); + + // Assign stack slots for any callee-saved registers that must be spilled. + assignCalleeSavedSpillSlots(Fn, SavedRegs); + + // Add the code to save and restore the callee saved registers. + if (!Fn.getFunction()->hasFnAttribute(Attribute::Naked)) + insertCSRSpillsAndRestores(Fn); + + finalizeFrame(Fn); + + // If register scavenging is needed, as we've enabled doing it as a + // post-pass, scavenge the virtual registers that frame index elimination + // inserted. + if (RS && FrameIndexVirtualScavenging) scavengeFrameVirtualRegs(Fn); + + // Clear any vregs created by virtual scavenging. + Fn.getRegInfo().clearVirtRegs(); + + delete RS; return true; } @@ -290,10 +350,10 @@ } } -void PEI::assignCalleeSavedSpillSlots(MachineFunction &F, - const BitVector &SavedRegs) { +void PEIWithCSRs::assignCalleeSavedSpillSlots(MachineFunction &F, + const BitVector &SavedRegs) { // These are used to keep track the callee-save area. Initialize them. - MinCSFrameIndex = INT_MAX; + MinCSFrameIndex = std::numeric_limits::max(); MaxCSFrameIndex = 0; if (SavedRegs.empty()) @@ -427,7 +487,7 @@ /// insertCSRSpillsAndRestores - Insert spill and restore code for /// callee saved registers used in the function. /// -void PEI::insertCSRSpillsAndRestores(MachineFunction &Fn) { +void PEIWithCSRs::insertCSRSpillsAndRestores(MachineFunction &Fn) { // Get callee saved register information. MachineFrameInfo *MFI = Fn.getFrameInfo(); const std::vector &CSI = MFI->getCalleeSavedInfo(); @@ -601,8 +661,8 @@ MFI->setObjectOffset(i, -Offset); // Set the computed offset } } else { - int MaxCSFI = MaxCSFrameIndex, MinCSFI = MinCSFrameIndex; - for (int i = MaxCSFI; i >= MinCSFI ; --i) { + unsigned MaxCSFI = MaxCSFrameIndex, MinCSFI = MinCSFrameIndex; + for (unsigned i = MaxCSFI; i >= MinCSFI; --i) { unsigned Align = MFI->getObjectAlignment(i); // Adjust to alignment boundary Offset = alignTo(Offset, Align, Skew); @@ -965,8 +1025,7 @@ /// FIXME: Iterating over the instruction stream is unnecessary. We can simply /// iterate over the vreg use list, which at this point only contains machine /// operands for which eliminateFrameIndex need a new scratch reg. -void -PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) { +void PEIWithCSRs::scavengeFrameVirtualRegs(MachineFunction &Fn) { // Run through the instructions and find any virtual registers. for (MachineFunction::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB) { @@ -1018,7 +1077,7 @@ // scratch register. assert (ScratchReg && "Missing scratch register!"); Fn.getRegInfo().replaceRegWith(Reg, ScratchReg); - + // Because this instruction was processed by the RS before this // register was allocated, make sure that the RS now records the // register as being used. Index: lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp =================================================================== --- lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp +++ lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp @@ -174,10 +174,11 @@ // TODO: The following CodeGen passes don't currently support code containing // virtual registers. Consider removing their restrictions and re-enabling // them. - // - // We use our own PrologEpilogInserter which is very slightly modified to - // tolerate virtual registers. - disablePass(&PrologEpilogCodeInserterID); + + // Use the VReg version of PrologEpilogInserter which doesn't support CSRs + // but does support virtual registers. + // TODO: Delete WebAssemblyPEI code. + substitutePass(&PrologEpilogCodeInserterID, &VRegPrologEpilogCodeInserterID); // Fails with: should be run after register allocation. disablePass(&MachineCopyPropagationID); @@ -188,11 +189,6 @@ addPass(createWebAssemblyRegColoring()); TargetPassConfig::addPostRegAlloc(); - - // Run WebAssembly's version of the PrologEpilogInserter. Target-independent - // PEI runs after PostRegAlloc and after ShrinkWrap. Putting it here will run - // PEI before ShrinkWrap but otherwise in the same position in the order. - addPass(createWebAssemblyPEI()); } void WebAssemblyPassConfig::addPreEmitPass() {