Changeset View
Changeset View
Standalone View
Standalone View
llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
| Show First 20 Lines • Show All 875 Lines • ▼ Show 20 Lines | |||||
| } | } | ||||
| // Convert callee-save register save/restore instruction to do stack pointer | // Convert callee-save register save/restore instruction to do stack pointer | ||||
| // decrement/increment to allocate/deallocate the callee-save stack area by | // decrement/increment to allocate/deallocate the callee-save stack area by | ||||
| // converting store/load to use pre/post increment version. | // converting store/load to use pre/post increment version. | ||||
| static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec( | static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec( | ||||
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, | MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, | ||||
| const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, | const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, | ||||
| bool NeedsWinCFI, bool *HasWinCFI, bool InProlog = true) { | bool NeedsDwarfCFI, bool NeedsWinCFI, bool *HasWinCFI, | ||||
| bool InProlog = true) { | |||||
| // Ignore instructions that do not operate on SP, i.e. shadow call stack | // Ignore instructions that do not operate on SP, i.e. shadow call stack | ||||
| // instructions and associated CFI instruction. | // instructions and associated CFI instruction. | ||||
| while (MBBI->getOpcode() == AArch64::STRXpost || | while (MBBI->getOpcode() == AArch64::STRXpost || | ||||
| MBBI->getOpcode() == AArch64::LDRXpre || | MBBI->getOpcode() == AArch64::LDRXpre || | ||||
| MBBI->getOpcode() == AArch64::CFI_INSTRUCTION) { | MBBI->getOpcode() == AArch64::CFI_INSTRUCTION) { | ||||
| if (MBBI->getOpcode() != AArch64::CFI_INSTRUCTION) | if (MBBI->getOpcode() != AArch64::CFI_INSTRUCTION) | ||||
| assert(MBBI->getOperand(0).getReg() != AArch64::SP); | assert(MBBI->getOperand(0).getReg() != AArch64::SP); | ||||
| ++MBBI; | ++MBBI; | ||||
| ▲ Show 20 Lines • Show All 80 Lines • ▼ Show 20 Lines | static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec( | ||||
| assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP && | assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP && | ||||
| "Unexpected base register in callee-save save/restore instruction!"); | "Unexpected base register in callee-save save/restore instruction!"); | ||||
| assert(CSStackSizeInc % Scale == 0); | assert(CSStackSizeInc % Scale == 0); | ||||
| MIB.addImm(CSStackSizeInc / (int)Scale); | MIB.addImm(CSStackSizeInc / (int)Scale); | ||||
| MIB.setMIFlags(MBBI->getFlags()); | MIB.setMIFlags(MBBI->getFlags()); | ||||
| MIB.setMemRefs(MBBI->memoperands()); | MIB.setMemRefs(MBBI->memoperands()); | ||||
| if (NeedsDwarfCFI && !InProlog) { | |||||
| unsigned CFIIndex = MBB.getParent()->addFrameInst( | |||||
| MCCFIInstruction::createAdjustCfaOffset(nullptr, -CSStackSizeInc)); | |||||
| BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) | |||||
| .addCFIIndex(CFIIndex); | |||||
| } | |||||
| // Generate a new SEH code that corresponds to the new instruction. | // Generate a new SEH code that corresponds to the new instruction. | ||||
| if (NeedsWinCFI) { | if (NeedsWinCFI) { | ||||
| *HasWinCFI = true; | *HasWinCFI = true; | ||||
| InsertSEH(*MIB, *TII, | InsertSEH(*MIB, *TII, | ||||
| InProlog ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy); | InProlog ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy); | ||||
| } | } | ||||
| return std::prev(MBB.erase(MBBI)); | return std::prev(MBB.erase(MBBI)); | ||||
| } | } | ||||
| // Fixup callee-save register save/restore instructions to take into account | // Fixup callee-save register save/restore instructions to take into account | ||||
| // combined SP bump by adding the local stack size to the stack offsets. | // combined SP bump by adding the local stack size to the stack offsets. | ||||
| static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, | static bool fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, | ||||
| uint64_t LocalStackSize, | uint64_t LocalStackSize, | ||||
| bool NeedsWinCFI, | bool NeedsWinCFI, | ||||
| bool *HasWinCFI) { | bool *HasWinCFI) { | ||||
| if (AArch64InstrInfo::isSEHInstruction(MI)) | if (AArch64InstrInfo::isSEHInstruction(MI)) | ||||
| return; | return false; | ||||
| unsigned Opc = MI.getOpcode(); | unsigned Opc = MI.getOpcode(); | ||||
| // Ignore instructions that do not operate on SP, i.e. shadow call stack | // Ignore instructions that do not operate on SP, i.e. shadow call stack | ||||
| // instructions and associated CFI instruction. | // instructions and associated CFI instruction. | ||||
| if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre || | if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre || | ||||
| Opc == AArch64::CFI_INSTRUCTION) { | Opc == AArch64::CFI_INSTRUCTION) { | ||||
| if (Opc != AArch64::CFI_INSTRUCTION) | if (Opc != AArch64::CFI_INSTRUCTION) | ||||
| assert(MI.getOperand(0).getReg() != AArch64::SP); | assert(MI.getOperand(0).getReg() != AArch64::SP); | ||||
| return; | return false; | ||||
| } | } | ||||
| unsigned Scale; | unsigned Scale; | ||||
| switch (Opc) { | switch (Opc) { | ||||
| case AArch64::STPXi: | case AArch64::STPXi: | ||||
| case AArch64::STRXui: | case AArch64::STRXui: | ||||
| case AArch64::STPDi: | case AArch64::STPDi: | ||||
| case AArch64::STRDui: | case AArch64::STRDui: | ||||
| Show All 25 Lines | static bool fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, | ||||
| if (NeedsWinCFI) { | if (NeedsWinCFI) { | ||||
| *HasWinCFI = true; | *HasWinCFI = true; | ||||
| auto MBBI = std::next(MachineBasicBlock::iterator(MI)); | auto MBBI = std::next(MachineBasicBlock::iterator(MI)); | ||||
| assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction"); | assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction"); | ||||
| assert(AArch64InstrInfo::isSEHInstruction(*MBBI) && | assert(AArch64InstrInfo::isSEHInstruction(*MBBI) && | ||||
| "Expecting a SEH instruction"); | "Expecting a SEH instruction"); | ||||
| fixupSEHOpcode(MBBI, LocalStackSize); | fixupSEHOpcode(MBBI, LocalStackSize); | ||||
| } | } | ||||
| return true; | |||||
| } | } | ||||
| static void adaptForLdStOpt(MachineBasicBlock &MBB, | static void adaptForLdStOpt(MachineBasicBlock &MBB, | ||||
| MachineBasicBlock::iterator FirstSPPopI, | MachineBasicBlock::iterator FirstSPPopI, | ||||
| MachineBasicBlock::iterator LastPopI) { | MachineBasicBlock::iterator LastPopI) { | ||||
| // Sometimes (when we restore in the same order as we save), we can end up | // Sometimes (when we restore in the same order as we save), we can end up | ||||
| // with code like this: | // with code like this: | ||||
| // | // | ||||
| ▲ Show 20 Lines • Show All 183 Lines • ▼ Show 20 Lines | emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, | ||||
| StackOffset::getFixed(-NumBytes), TII, | StackOffset::getFixed(-NumBytes), TII, | ||||
| MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); | MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); | ||||
| NumBytes = 0; | NumBytes = 0; | ||||
| } else if (HomPrologEpilog) { | } else if (HomPrologEpilog) { | ||||
| // Stack has been already adjusted. | // Stack has been already adjusted. | ||||
| NumBytes -= PrologueSaveSize; | NumBytes -= PrologueSaveSize; | ||||
| } else if (PrologueSaveSize != 0) { | } else if (PrologueSaveSize != 0) { | ||||
| MBBI = convertCalleeSaveRestoreToSPPrePostIncDec( | MBBI = convertCalleeSaveRestoreToSPPrePostIncDec( | ||||
| MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI); | MBB, MBBI, DL, TII, -PrologueSaveSize, false, NeedsWinCFI, &HasWinCFI); | ||||
| NumBytes -= PrologueSaveSize; | NumBytes -= PrologueSaveSize; | ||||
| } | } | ||||
| assert(NumBytes >= 0 && "Negative stack allocation size!?"); | assert(NumBytes >= 0 && "Negative stack allocation size!?"); | ||||
| // Move past the saves of the callee-saved registers, fixing up the offsets | // Move past the saves of the callee-saved registers, fixing up the offsets | ||||
| // and pre-inc if we decided to combine the callee-save and local stack | // and pre-inc if we decided to combine the callee-save and local stack | ||||
| // pointer bump above. | // pointer bump above. | ||||
| MachineBasicBlock::iterator End = MBB.end(); | MachineBasicBlock::iterator End = MBB.end(); | ||||
| ▲ Show 20 Lines • Show All 413 Lines • ▼ Show 20 Lines | void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, | ||||
| if (MBB.end() != MBBI) { | if (MBB.end() != MBBI) { | ||||
| DL = MBBI->getDebugLoc(); | DL = MBBI->getDebugLoc(); | ||||
| IsFunclet = isFuncletReturnInstr(*MBBI); | IsFunclet = isFuncletReturnInstr(*MBBI); | ||||
| } | } | ||||
| int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF) | int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF) | ||||
| : MFI.getStackSize(); | : MFI.getStackSize(); | ||||
| AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); | AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); | ||||
| // TODO Also emit CFI when SVECalleeSavedStackSize != 0. | |||||
MaskRay: SVE callee save stack seems very difficult, so not in the scope of this patch. | |||||
| bool NeedsDwarfCFI = MF.getTarget().getTargetTriple().isOSBinFormatELF() && | |||||
| MF.getFunction().needsUnwindTableEntry() && | |||||
| AFI->getSVECalleeSavedStackSize() == 0; | |||||
| // All calls are tail calls in GHC calling conv, and functions have no | // All calls are tail calls in GHC calling conv, and functions have no | ||||
| // prologue/epilogue. | // prologue/epilogue. | ||||
| if (MF.getFunction().getCallingConv() == CallingConv::GHC) | if (MF.getFunction().getCallingConv() == CallingConv::GHC) | ||||
| return; | return; | ||||
| // How much of the stack used by incoming arguments this function is expected | // How much of the stack used by incoming arguments this function is expected | ||||
| // to restore in this particular epilogue. | // to restore in this particular epilogue. | ||||
| ▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines | if (!CombineSPBump && PrologueSaveSize != 0) { | ||||
| while (AArch64InstrInfo::isSEHInstruction(*Pop)) | while (AArch64InstrInfo::isSEHInstruction(*Pop)) | ||||
| Pop = std::prev(Pop); | Pop = std::prev(Pop); | ||||
| // Converting the last ldp to a post-index ldp is valid only if the last | // Converting the last ldp to a post-index ldp is valid only if the last | ||||
| // ldp's offset is 0. | // ldp's offset is 0. | ||||
| const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1); | const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1); | ||||
| // If the offset is 0 and the AfterCSR pop is not actually trying to | // If the offset is 0 and the AfterCSR pop is not actually trying to | ||||
| // allocate more stack for arguments (in space that an untimely interrupt | // allocate more stack for arguments (in space that an untimely interrupt | ||||
| // may clobber), convert it to a post-index ldp. | // may clobber), convert it to a post-index ldp. | ||||
| if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0) | if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0) { | ||||
| convertCalleeSaveRestoreToSPPrePostIncDec( | convertCalleeSaveRestoreToSPPrePostIncDec(MBB, Pop, DL, TII, | ||||
| MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, false); | PrologueSaveSize, NeedsDwarfCFI, | ||||
| else { | NeedsWinCFI, &HasWinCFI, false); | ||||
| } else { | |||||
| // If not, make sure to emit an add after the last ldp. | // If not, make sure to emit an add after the last ldp. | ||||
| // We're doing this by transfering the size to be restored from the | // We're doing this by transfering the size to be restored from the | ||||
| // adjustment *before* the CSR pops to the adjustment *after* the CSR | // adjustment *before* the CSR pops to the adjustment *after* the CSR | ||||
| // pops. | // pops. | ||||
| AfterCSRPopSize += PrologueSaveSize; | AfterCSRPopSize += PrologueSaveSize; | ||||
| } | } | ||||
| } | } | ||||
| bool EmitCfi = NeedsDwarfCFI && !hasFP(MF); | |||||
| // Move past the restores of the callee-saved registers. | // Move past the restores of the callee-saved registers. | ||||
| // If we plan on combining the sp bump of the local stack size and the callee | // If we plan on combining the sp bump of the local stack size and the callee | ||||
| // save stack size, we might need to adjust the CSR save and restore offsets. | // save stack size, we might need to adjust the CSR save and restore offsets. | ||||
| MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator(); | MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator(); | ||||
| MachineBasicBlock::iterator Begin = MBB.begin(); | MachineBasicBlock::iterator Begin = MBB.begin(); | ||||
| while (LastPopI != Begin) { | while (LastPopI != Begin) { | ||||
| --LastPopI; | --LastPopI; | ||||
| if (!LastPopI->getFlag(MachineInstr::FrameDestroy) || | if (!LastPopI->getFlag(MachineInstr::FrameDestroy) || | ||||
| IsSVECalleeSave(LastPopI)) { | IsSVECalleeSave(LastPopI)) { | ||||
| ++LastPopI; | ++LastPopI; | ||||
| break; | break; | ||||
| } else if (CombineSPBump) | } else if (CombineSPBump) { | ||||
Lint: Pre-merge checks clang-tidy: warning: do not use 'else' after 'break' [llvm-else-after-return] Lint: Pre-merge checks: clang-tidy: warning: do not use 'else' after 'break' [llvm-else-after-return]
[[https://github. | |||||
| fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize(), | if (fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize(), | ||||
| NeedsWinCFI, &HasWinCFI); | NeedsWinCFI, &HasWinCFI) && | ||||
| EmitCfi) { | |||||
| unsigned CFIIndex = | |||||
| MF.addFrameInst(MCCFIInstruction::createAdjustCfaOffset( | |||||
| nullptr, AFI->getLocalStackSize())); | |||||
| BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) | |||||
| .addCFIIndex(CFIIndex); | |||||
| } | |||||
| } | |||||
| } | } | ||||
| if (MF.hasWinCFI()) { | if (MF.hasWinCFI()) { | ||||
| // If the prologue didn't contain any SEH opcodes and didn't set the | // If the prologue didn't contain any SEH opcodes and didn't set the | ||||
| // MF.hasWinCFI() flag, assume the epilogue won't either, and skip the | // MF.hasWinCFI() flag, assume the epilogue won't either, and skip the | ||||
| // EpilogStart - to avoid generating CFI for functions that don't need it. | // EpilogStart - to avoid generating CFI for functions that don't need it. | ||||
| // (And as we didn't generate any prologue at all, it would be asymmetrical | // (And as we didn't generate any prologue at all, it would be asymmetrical | ||||
| // to the epilogue.) By the end of the function, we assert that | // to the epilogue.) By the end of the function, we assert that | ||||
| ▲ Show 20 Lines • Show All 1,899 Lines • Show Last 20 Lines | |||||
SVE callee save stack seems very difficult, so not in the scope of this patch.