Index: include/llvm/CodeGen/Passes.h =================================================================== --- include/llvm/CodeGen/Passes.h +++ include/llvm/CodeGen/Passes.h @@ -484,6 +484,7 @@ /// PrologEpilogCodeInserter - This pass inserts prolog and epilog code, /// and eliminates abstract frame references. extern char &PrologEpilogCodeInserterID; + MachineFunctionPass *createPrologEpilogInserterPass(const TargetMachine *TM); /// ExpandPostRAPseudos - This pass expands pseudo instructions after /// register allocation. Index: lib/CodeGen/Passes.cpp =================================================================== --- lib/CodeGen/Passes.cpp +++ lib/CodeGen/Passes.cpp @@ -559,7 +559,7 @@ if (getOptLevel() != CodeGenOpt::None) addPass(&ShrinkWrapID); - addPass(&PrologEpilogCodeInserterID); + addPass(createPrologEpilogInserterPass(TM)); /// Add passes that optimize machine instructions after register allocation. if (getOptLevel() != CodeGenOpt::None) Index: lib/CodeGen/PrologEpilogInserter.cpp =================================================================== --- lib/CodeGen/PrologEpilogInserter.cpp +++ lib/CodeGen/PrologEpilogInserter.cpp @@ -49,48 +49,64 @@ #define DEBUG_TYPE "pei" +using MBBVector = SmallVector; +static void DoSpillCalleeSavedRegs(MachineFunction &MF, RegScavenger *RS, + unsigned &MinCSFrameIndex, + unsigned &MaxCXFrameIndex, + const MBBVector &SaveBlocks, + const MBBVector &RestoreBlocks); + namespace { class PEI : public MachineFunctionPass { public: static char ID; - PEI() : MachineFunctionPass(ID) { + explicit PEI(const TargetMachine *TM = nullptr) : MachineFunctionPass(ID) { initializePEIPass(*PassRegistry::getPassRegistry()); + + // XXX replace this with some sort of target hook + if (TM->getTargetTriple().getArch() == Triple::wasm32) { + SpillCalleeSavedRegisters = [](MachineFunction &, RegScavenger *, + unsigned &, unsigned &, const MBBVector &, + const MBBVector &) {}; + } else { + SpillCalleeSavedRegisters = DoSpillCalleeSavedRegs; + } } void getAnalysisUsage(AnalysisUsage &AU) const override; - MachineFunctionProperties getRequiredProperties() const override { - return MachineFunctionProperties().set( - MachineFunctionProperties::Property::AllVRegsAllocated); - } - /// runOnMachineFunction - Insert prolog/epilog code and replace abstract /// frame indexes with appropriate references. /// bool runOnMachineFunction(MachineFunction &Fn) override; private: + std::function + SpillCalleeSavedRegisters; + RegScavenger *RS; // MinCSFrameIndex, MaxCSFrameIndex - Keeps the range of callee saved // stack frame indexes. - unsigned MinCSFrameIndex, MaxCSFrameIndex; + unsigned MinCSFrameIndex = std::numeric_limits::max(); + unsigned MaxCSFrameIndex = 0; // Save and Restore blocks of the current function. Typically there is a // single save block, unless Windows EH funclets are involved. - SmallVector SaveBlocks; - SmallVector RestoreBlocks; + MBBVector SaveBlocks; + MBBVector RestoreBlocks; // Flag to control whether to use the register scavenger to resolve // frame index materialization registers. Set according to // TRI->requiresFrameIndexScavenging() for the current function. bool FrameIndexVirtualScavenging; - void calculateSets(MachineFunction &Fn); - void calculateCallsInformation(MachineFunction &Fn); - void assignCalleeSavedSpillSlots(MachineFunction &Fn, - const BitVector &SavedRegs); - void insertCSRSpillsAndRestores(MachineFunction &Fn); + void calculateCallFrameInfo(MachineFunction &Fn); + void calculateSaveRestoreBlocks(MachineFunction &Fn); + void calculateFrameObjectOffsets(MachineFunction &Fn); void replaceFrameIndices(MachineFunction &Fn); void replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn, @@ -108,15 +124,20 @@ cl::desc("Warn for stack size bigger than the given" " number")); -INITIALIZE_PASS_BEGIN(PEI, "prologepilog", - "Prologue/Epilogue Insertion", false, false) +INITIALIZE_TM_PASS_BEGIN(PEI, "prologepilog", "Prologue/Epilogue Insertion", + false, false) INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) INITIALIZE_PASS_DEPENDENCY(StackProtector) INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) -INITIALIZE_PASS_END(PEI, "prologepilog", - "Prologue/Epilogue Insertion & Frame Finalization", - false, false) +INITIALIZE_TM_PASS_END(PEI, "prologepilog", + "Prologue/Epilogue Insertion & Frame Finalization", + false, false) + +MachineFunctionPass * +llvm::createPrologEpilogInserterPass(const TargetMachine *TM) { + return new PEI(TM); +} STATISTIC(NumScavengedRegs, "Number of frame index regs scavenged"); STATISTIC(NumBytesStackSpace, @@ -131,36 +152,6 @@ MachineFunctionPass::getAnalysisUsage(AU); } -/// Compute the set of return blocks -void PEI::calculateSets(MachineFunction &Fn) { - const MachineFrameInfo *MFI = Fn.getFrameInfo(); - - // Even when we do not change any CSR, we still want to insert the - // prologue and epilogue of the function. - // So set the save points for those. - - // Use the points found by shrink-wrapping, if any. - if (MFI->getSavePoint()) { - SaveBlocks.push_back(MFI->getSavePoint()); - assert(MFI->getRestorePoint() && "Both restore and save must be set"); - MachineBasicBlock *RestoreBlock = MFI->getRestorePoint(); - // If RestoreBlock does not have any successor and is not a return block - // then the end point is unreachable and we do not need to insert any - // epilogue. - if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock()) - RestoreBlocks.push_back(RestoreBlock); - return; - } - - // Save refs to entry and return blocks. - SaveBlocks.push_back(&Fn.front()); - for (MachineBasicBlock &MBB : Fn) { - if (MBB.isEHFuncletEntry()) - SaveBlocks.push_back(&MBB); - if (MBB.isReturnBlock()) - RestoreBlocks.push_back(&MBB); - } -} /// StackObjSet - A set of stack object indexes typedef SmallSetVector StackObjSet; @@ -173,30 +164,26 @@ const TargetRegisterInfo *TRI = Fn.getSubtarget().getRegisterInfo(); const TargetFrameLowering *TFI = Fn.getSubtarget().getFrameLowering(); - assert(!Fn.getRegInfo().getNumVirtRegs() && "Regalloc must assign all vregs"); - RS = TRI->requiresRegisterScavenging(Fn) ? new RegScavenger() : nullptr; FrameIndexVirtualScavenging = TRI->requiresFrameIndexScavenging(Fn); + // Virtual-register machines such as WebAssembly do not use normal register + // allocation. Such targets also do not use callee-save registers and do + // not require register scavenging. + const bool HasVirtualRegisters = Fn.getRegInfo().getNumVirtRegs(); + // Calculate the MaxCallFrameSize and AdjustsStack variables for the // function's frame information. Also eliminates call frame pseudo // instructions. - calculateCallsInformation(Fn); - - // Determine which of the registers in the callee save list should be saved. - BitVector SavedRegs; - TFI->determineCalleeSaves(Fn, SavedRegs, RS); + calculateCallFrameInfo(Fn); - // Insert spill code for any callee saved registers that are modified. - assignCalleeSavedSpillSlots(Fn, SavedRegs); - - // Determine placement of CSR spill/restore code: + // Determine placement of CSR spill/restore code and prolog/epilog code: // place all spills in the entry block, all restores in return blocks. - calculateSets(Fn); + calculateSaveRestoreBlocks(Fn); - // Add the code to save and restore the callee saved registers. - if (!F->hasFnAttribute(Attribute::Naked)) - insertCSRSpillsAndRestores(Fn); + // Handle CSR spilling and restoring, for targets that need it. + SpillCalleeSavedRegisters(Fn, RS, MinCSFrameIndex, MaxCSFrameIndex, + SaveBlocks, RestoreBlocks); // Allow the target machine to make final modifications to the function // before the frame layout is finalized. @@ -218,14 +205,19 @@ // replaceFrameIndices(Fn); - // If register scavenging is needed, as we've enabled doing it as a - // post-pass, scavenge the virtual registers that frame index elimination - // inserted. - if (TRI->requiresRegisterScavenging(Fn) && FrameIndexVirtualScavenging) - scavengeFrameVirtualRegs(Fn); - - // Clear any vregs created by virtual scavenging. - Fn.getRegInfo().clearVirtRegs(); + if (HasVirtualRegisters) { + assert(!RS && !TRI->requiresFrameIndexScavenging(Fn) && + "Virtual-register targets do not support register scavenging"); + } else { + // If register scavenging is needed, as we've enabled doing it as a + // post-pass, scavenge the virtual registers that frame index elimination + // inserted. + if (TRI->requiresRegisterScavenging(Fn) && FrameIndexVirtualScavenging) + scavengeFrameVirtualRegs(Fn); + + // Clear any vregs created by virtual scavenging. + Fn.getRegInfo().clearVirtRegs(); + } // Warn on stack size when we exceeds the given limit. MachineFrameInfo *MFI = Fn.getFrameInfo(); @@ -241,10 +233,10 @@ return true; } -/// calculateCallsInformation - Calculate the MaxCallFrameSize and AdjustsStack +/// Calculate the MaxCallFrameSize and AdjustsStack /// variables for the function's frame information and eliminate call frame /// pseudo instructions. -void PEI::calculateCallsInformation(MachineFunction &Fn) { +void PEI::calculateCallFrameInfo(MachineFunction &Fn) { const TargetInstrInfo &TII = *Fn.getSubtarget().getInstrInfo(); const TargetFrameLowering *TFI = Fn.getSubtarget().getFrameLowering(); MachineFrameInfo *MFI = Fn.getFrameInfo(); @@ -295,12 +287,42 @@ } } -void PEI::assignCalleeSavedSpillSlots(MachineFunction &F, - const BitVector &SavedRegs) { - // These are used to keep track the callee-save area. Initialize them. - MinCSFrameIndex = INT_MAX; - MaxCSFrameIndex = 0; +/// Compute the sets of entry and return blocks for saving and restoring +/// callee-saved registers, and placing prolog and epilog code. +void PEI::calculateSaveRestoreBlocks(MachineFunction &Fn) { + const MachineFrameInfo *MFI = Fn.getFrameInfo(); + + // Even when we do not change any CSR, we still want to insert the + // prologue and epilogue of the function. + // So set the save points for those. + + // Use the points found by shrink-wrapping, if any. + if (MFI->getSavePoint()) { + SaveBlocks.push_back(MFI->getSavePoint()); + assert(MFI->getRestorePoint() && "Both restore and save must be set"); + MachineBasicBlock *RestoreBlock = MFI->getRestorePoint(); + // If RestoreBlock does not have any successor and is not a return block + // then the end point is unreachable and we do not need to insert any + // epilogue. + if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock()) + RestoreBlocks.push_back(RestoreBlock); + return; + } + + // Save refs to entry and return blocks. + SaveBlocks.push_back(&Fn.front()); + for (MachineBasicBlock &MBB : Fn) { + if (MBB.isEHFuncletEntry()) + SaveBlocks.push_back(&MBB); + if (MBB.isReturnBlock()) + RestoreBlocks.push_back(&MBB); + } +} +static void assignCalleeSavedSpillSlots(MachineFunction &F, + const BitVector &SavedRegs, + unsigned &MinCSFrameIndex, + unsigned &MaxCSFrameIndex) { if (SavedRegs.empty()) return; @@ -431,7 +453,9 @@ /// insertCSRSpillsAndRestores - Insert spill and restore code for /// callee saved registers used in the function. /// -void PEI::insertCSRSpillsAndRestores(MachineFunction &Fn) { +static void insertCSRSpillsAndRestores(MachineFunction &Fn, + const MBBVector &SaveBlocks, + const MBBVector &RestoreBlocks) { // Get callee saved register information. MachineFrameInfo *MFI = Fn.getFrameInfo(); const std::vector &CSI = MFI->getCalleeSavedInfo(); @@ -500,6 +524,25 @@ } } +static void DoSpillCalleeSavedRegs(MachineFunction &Fn, RegScavenger *RS, + unsigned &MinCSFrameIndex, + unsigned &MaxCSFrameIndex, + const MBBVector &SaveBlocks, + const MBBVector &RestoreBlocks) { + const Function *F = Fn.getFunction(); + const TargetFrameLowering *TFI = Fn.getSubtarget().getFrameLowering(); + // Determine which of the registers in the callee save list should be saved. + BitVector SavedRegs; + TFI->determineCalleeSaves(Fn, SavedRegs, RS); + + // Assign stack slots for any callee-saved registers that must be spilled. + assignCalleeSavedSpillSlots(Fn, SavedRegs, MinCSFrameIndex, MaxCSFrameIndex); + + // Add the code to save and restore the callee saved registers. + if (!F->hasFnAttribute(Attribute::Naked)) + insertCSRSpillsAndRestores(Fn, SaveBlocks, RestoreBlocks); +} + /// AdjustStackOffset - Helper function used to adjust the stack frame offset. static inline void AdjustStackOffset(MachineFrameInfo *MFI, int FrameIdx, @@ -606,8 +649,8 @@ MFI->setObjectOffset(i, -Offset); // Set the computed offset } } else { - int MaxCSFI = MaxCSFrameIndex, MinCSFI = MinCSFrameIndex; - for (int i = MaxCSFI; i >= MinCSFI ; --i) { + unsigned MaxCSFI = MaxCSFrameIndex, MinCSFI = MinCSFrameIndex; + for (unsigned i = MaxCSFI; i >= MinCSFI; --i) { unsigned Align = MFI->getObjectAlignment(i); // Adjust to alignment boundary Offset = alignTo(Offset, Align, Skew); Index: lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp =================================================================== --- lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp +++ lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp @@ -182,9 +182,6 @@ // Has no asserts of its own, but was not written to handle virtual regs. disablePass(&ShrinkWrapID); - // We use our own PrologEpilogInserter which is very slightly modified to - // tolerate virtual registers. - disablePass(&PrologEpilogCodeInserterID); // These functions all require the AllVRegsAllocated property. disablePass(&MachineCopyPropagationID); @@ -202,11 +199,6 @@ } TargetPassConfig::addPostRegAlloc(); - - // Run WebAssembly's version of the PrologEpilogInserter. Target-independent - // PEI runs after PostRegAlloc and after ShrinkWrap. Putting it here will run - // PEI before ShrinkWrap but otherwise in the same position in the order. - addPass(createWebAssemblyPEI()); } void WebAssemblyPassConfig::addPreEmitPass() {