Index: llvm/lib/Target/AArch64/AArch64FrameLowering.h =================================================================== --- llvm/lib/Target/AArch64/AArch64FrameLowering.h +++ llvm/lib/Target/AArch64/AArch64FrameLowering.h @@ -148,6 +148,10 @@ MachineBasicBlock::iterator MBBI) const; void emitCalleeSavedSVELocations(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const; + void emitCalleeSavedGPRRestores(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI) const; + void emitCalleeSavedSVERestores(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI) const; }; } // End llvm namespace Index: llvm/lib/Target/AArch64/AArch64FrameLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64FrameLowering.cpp +++ llvm/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -579,6 +579,49 @@ emitCalleeSavedSVELocations(MBB, MBBI); } +static void emitCalleeSavedRestores(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + bool SVE) { + MachineFunction &MF = *MBB.getParent(); + MachineFrameInfo &MFI = MF.getFrameInfo(); + + const std::vector &CSI = MFI.getCalleeSavedInfo(); + if (CSI.empty()) + return; + + const TargetSubtargetInfo &STI = MF.getSubtarget(); + const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); + const TargetInstrInfo &TII = *STI.getInstrInfo(); + DebugLoc DL = MBB.findDebugLoc(MBBI); + + for (const auto &Info : CSI) { + if (SVE != + (MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector)) + continue; + + unsigned Reg = Info.getReg(); + if (SVE && + !static_cast(TRI).regNeedsCFI(Reg, Reg)) + continue; + + unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createRestore( + nullptr, TRI.getDwarfRegNum(Info.getReg(), true))); + BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION)) + .addCFIIndex(CFIIndex) + .setMIFlags(MachineInstr::FrameDestroy); + } +} + +void AArch64FrameLowering::emitCalleeSavedGPRRestores( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { + emitCalleeSavedRestores(MBB, MBBI, false); +} + +void AArch64FrameLowering::emitCalleeSavedSVERestores( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { + emitCalleeSavedRestores(MBB, MBBI, true); +} + // Find a scratch register that we can use at the start of the prologue to // re-align the stack pointer. We avoid using callee-save registers since they // may appear to be free when this is called from canUseAsPrologue (during @@ -886,7 +929,9 @@ static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, - bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI, bool InProlog = true) { + bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI, + MachineInstr::MIFlag FrameFlag = MachineInstr::FrameSetup, + int CFAOffset = 0) { unsigned NewOpc; switch (MBBI->getOpcode()) { default: @@ -949,10 +994,9 @@ if (MBBI->getOperand(MBBI->getNumOperands() - 1).getImm() != 0 || CSStackSizeInc < MinOffset || CSStackSizeInc > MaxOffset) { emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, - StackOffset::getFixed(CSStackSizeInc), TII, - InProlog ? MachineInstr::FrameSetup - : MachineInstr::FrameDestroy, - false, false, nullptr, EmitCFI && InProlog); + StackOffset::getFixed(CSStackSizeInc), TII, FrameFlag, + false, false, nullptr, EmitCFI, + StackOffset::getFixed(CFAOffset)); return std::prev(MBBI); } @@ -980,16 +1024,15 @@ // Generate a new SEH code that corresponds to the new instruction. if (NeedsWinCFI) { *HasWinCFI = true; - InsertSEH(*MIB, *TII, - InProlog ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy); + InsertSEH(*MIB, *TII, FrameFlag); } - if (EmitCFI && InProlog) { + if (EmitCFI) { unsigned CFIIndex = MF.addFrameInst( - MCCFIInstruction::cfiDefCfaOffset(nullptr, -CSStackSizeInc)); + MCCFIInstruction::cfiDefCfaOffset(nullptr, CFAOffset - CSStackSizeInc)); BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) .addCFIIndex(CFIIndex) - .setMIFlags(MachineInstr::FrameSetup); + .setMIFlags(FrameFlag); } return std::prev(MBB.erase(MBBI)); @@ -1128,6 +1171,14 @@ .addReg(AArch64::X18) .addImm(-8) .setMIFlag(MachineInstr::FrameDestroy); + + if (MF.getInfo()->needsAsyncDwarfUnwindInfo()) { + unsigned CFIIndex = + MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, 18)); + BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION)) + .addCFIIndex(CFIIndex) + .setMIFlags(MachineInstr::FrameDestroy); + } } void AArch64FrameLowering::emitPrologue(MachineFunction &MF, @@ -1163,7 +1214,6 @@ MFnI.needsDwarfUnwindInfo()); if (MFnI.shouldSignReturnAddress()) { - unsigned PACI; if (MFnI.shouldSignWithBKey()) { BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITBKEY)) @@ -1657,6 +1707,7 @@ const TargetInstrInfo *TII = Subtarget.getInstrInfo(); DebugLoc DL; bool NeedsWinCFI = needsWinCFI(MF); + bool EmitCFI = MF.getInfo()->needsAsyncDwarfUnwindInfo(); bool HasWinCFI = false; bool IsFunclet = false; auto WinCFI = make_scope_exit([&]() { assert(HasWinCFI == MF.hasWinCFI()); }); @@ -1666,9 +1717,12 @@ IsFunclet = isFuncletReturnInstr(*MBBI); } - auto ShadowStackEpilogue = make_scope_exit([&]() { + auto FinishingTouches = make_scope_exit([&]() { + InsertReturnAddressAuth(MF, MBB); if (needsShadowCallStackPrologueEpilogue(MF)) emitShadowCallStackEpilogue(*TII, MF, MBB, MBB.getFirstTerminator(), DL); + if (EmitCFI) + emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator()); }); int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF) @@ -1683,36 +1737,6 @@ // How much of the stack used by incoming arguments this function is expected // to restore in this particular epilogue. int64_t ArgumentStackToRestore = getArgumentStackToRestore(MF, MBB); - - // The stack frame should be like below, - // - // ---------------------- --- - // | | | - // | BytesInStackArgArea| CalleeArgStackSize - // | (NumReusableBytes) | (of tail call) - // | | --- - // | | | - // ---------------------| --- | - // | | | | - // | CalleeSavedReg | | | - // | (CalleeSavedStackSize)| | | - // | | | | - // ---------------------| | NumBytes - // | | StackSize (StackAdjustUp) - // | LocalStackSize | | | - // | (covering callee | | | - // | args) | | | - // | | | | - // ---------------------- --- --- - // - // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize - // = StackSize + ArgumentPopSize - // - // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps - // it as the 2nd argument of AArch64ISD::TC_RETURN. - - auto Cleanup = make_scope_exit([&] { InsertReturnAddressAuth(MF, MBB); }); - bool IsWin64 = Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()); unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet); @@ -1747,9 +1771,11 @@ bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(MBB, NumBytes); // Assume we can't combine the last pop with the sp restore. + bool CombineAfterCSRBump = false; if (!CombineSPBump && PrologueSaveSize != 0) { MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator()); - while (AArch64InstrInfo::isSEHInstruction(*Pop)) + while (Pop->getOpcode() == TargetOpcode::CFI_INSTRUCTION || + AArch64InstrInfo::isSEHInstruction(*Pop)) Pop = std::prev(Pop); // Converting the last ldp to a post-index ldp is valid only if the last // ldp's offset is 0. @@ -1757,16 +1783,17 @@ // If the offset is 0 and the AfterCSR pop is not actually trying to // allocate more stack for arguments (in space that an untimely interrupt // may clobber), convert it to a post-index ldp. - if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0) + if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0) { convertCalleeSaveRestoreToSPPrePostIncDec( MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, - false, false); - else { + EmitCFI, MachineInstr::FrameDestroy, PrologueSaveSize); + } else { // If not, make sure to emit an add after the last ldp. // We're doing this by transfering the size to be restored from the // adjustment *before* the CSR pops to the adjustment *after* the CSR // pops. AfterCSRPopSize += PrologueSaveSize; + CombineAfterCSRBump = true; } } @@ -1815,14 +1842,30 @@ // If there is a single SP update, insert it before the ret and we're done. if (CombineSPBump) { assert(!SVEStackSize && "Cannot combine SP bump with SVE"); + + // When we are about to restore the CSRs, the CFA register is SP again. + if (EmitCFI && hasFP(MF)) { + const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo(); + unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true); + unsigned CFIIndex = MF.addFrameInst( + MCCFIInstruction::cfiDefCfa(nullptr, Reg, NumBytes)); + BuildMI(MBB, LastPopI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) + .addCFIIndex(CFIIndex) + .setMIFlags(MachineInstr::FrameDestroy); + } + + emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP, StackOffset::getFixed(NumBytes + (int64_t)AfterCSRPopSize), TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, - &HasWinCFI); + &HasWinCFI, EmitCFI, + StackOffset::getFixed(NumBytes)); if (HasWinCFI) BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd)) .setMIFlag(MachineInstr::FrameDestroy); + // if (EmitCFI) + // emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator()); return; } @@ -1851,29 +1894,40 @@ // Deallocate the SVE area. if (SVEStackSize) { if (AFI->isStackRealigned()) { - if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) + if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) { // Set SP to start of SVE callee-save area from which they can // be reloaded. The code below will deallocate the stack space // space by moving FP -> SP. emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::FP, StackOffset::getScalable(-CalleeSavedSize), TII, MachineInstr::FrameDestroy); + } } else { if (AFI->getSVECalleeSavedStackSize()) { // Deallocate the non-SVE locals first before we can deallocate (and // restore callee saves) from the SVE area. - emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP, - StackOffset::getFixed(NumBytes), TII, - MachineInstr::FrameDestroy); + emitFrameOffset( + MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP, + StackOffset::getFixed(NumBytes), TII, MachineInstr::FrameDestroy, + false, false, nullptr, EmitCFI && !hasFP(MF), + SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize)); NumBytes = 0; } emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP, - DeallocateBefore, TII, MachineInstr::FrameDestroy); + DeallocateBefore, TII, MachineInstr::FrameDestroy, false, + false, nullptr, EmitCFI && !hasFP(MF), + SVEStackSize + + StackOffset::getFixed(NumBytes + PrologueSaveSize)); emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP, - DeallocateAfter, TII, MachineInstr::FrameDestroy); + DeallocateAfter, TII, MachineInstr::FrameDestroy, false, + false, nullptr, EmitCFI && !hasFP(MF), + DeallocateAfter + + StackOffset::getFixed(NumBytes + PrologueSaveSize)); } + if (EmitCFI) + emitCalleeSavedSVERestores(MBB, RestoreEnd); } if (!hasFP(MF)) { @@ -1883,14 +1937,21 @@ if (RedZone && AfterCSRPopSize == 0) return; + // Pop the local variables off the stack. If there are no callee-saved + // registers, it means we are actually positioned at the terminator and can + // combine stack increment for the locals and the stack increment for + // callee-popped arguments into (possibly) a single instruction and be done. bool NoCalleeSaveRestore = PrologueSaveSize == 0; int64_t StackRestoreBytes = RedZone ? 0 : NumBytes; if (NoCalleeSaveRestore) StackRestoreBytes += AfterCSRPopSize; - emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, - StackOffset::getFixed(StackRestoreBytes), TII, - MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI); + emitFrameOffset( + MBB, LastPopI, DL, AArch64::SP, AArch64::SP, + StackOffset::getFixed(StackRestoreBytes), TII, + MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI, + StackOffset::getFixed((RedZone ? 0 : NumBytes) + PrologueSaveSize)); + // If we were able to combine the local stack pop with the argument pop, // then we're done. if (NoCalleeSaveRestore || AfterCSRPopSize == 0) { @@ -1899,6 +1960,10 @@ TII->get(AArch64::SEH_EpilogEnd)) .setMIFlag(MachineInstr::FrameDestroy); } + + // if (EmitCFI) + // emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator()); + return; } @@ -1919,6 +1984,17 @@ StackOffset::getFixed(NumBytes), TII, MachineInstr::FrameDestroy, false, NeedsWinCFI); + // When we are about to restore the CSRs, the CFA register is SP again. + if (EmitCFI && hasFP(MF)) { + const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo(); + unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true); + unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa( + nullptr, Reg, PrologueSaveSize)); + BuildMI(MBB, LastPopI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) + .addCFIIndex(CFIIndex) + .setMIFlags(MachineInstr::FrameDestroy); + } + // This must be placed after the callee-save restore code because that code // assumes the SP is at the same location as it was after the callee-save save // code in the prologue. @@ -1928,11 +2004,14 @@ emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP, StackOffset::getFixed(AfterCSRPopSize), TII, - MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI); + MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, + EmitCFI, StackOffset::getFixed(CombineAfterCSRBump ? PrologueSaveSize : 0)); } if (HasWinCFI) BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd)) .setMIFlag(MachineInstr::FrameDestroy); + // if (EmitCFI) + // emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator()); } /// getFrameIndexReference - Provide a base+offset reference to an FI slot for @@ -2624,6 +2703,7 @@ MachineMemOperand::MOLoad, Size, Alignment)); if (NeedsWinCFI) InsertSEH(MIB, TII, MachineInstr::FrameDestroy); + return MIB->getIterator(); }; Index: llvm/lib/Target/AArch64/AArch64InstrInfo.h =================================================================== --- llvm/lib/Target/AArch64/AArch64InstrInfo.h +++ llvm/lib/Target/AArch64/AArch64InstrInfo.h @@ -364,7 +364,8 @@ const TargetRegisterInfo *TRI); MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg, - unsigned Reg, const StackOffset &Offset); + unsigned Reg, const StackOffset &Offset, + bool LastAdjustmentWasScalable = true); MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg, const StackOffset &OffsetFromDefCFA); Index: llvm/lib/Target/AArch64/AArch64InstrInfo.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -4064,11 +4064,12 @@ MCCFIInstruction llvm::createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg, unsigned Reg, - const StackOffset &Offset) { + const StackOffset &Offset, + bool LastAdjustmentWasScalable) { if (Offset.getScalable()) return createDefCFAExpression(TRI, Reg, Offset); - if (FrameReg == Reg) + if (FrameReg == Reg && !LastAdjustmentWasScalable) return MCCFIInstruction::cfiDefCfaOffset(nullptr, int(Offset.getFixed())); unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true); @@ -4200,7 +4201,7 @@ const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); unsigned CFIIndex = - MF.addFrameInst(createDefCFA(TRI, FrameReg, DestReg, CFAOffset)); + MF.addFrameInst(createDefCFA(TRI, FrameReg, DestReg, CFAOffset, VScale != 1)); BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) .addCFIIndex(CFIIndex) .setMIFlags(Flag); Index: llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h =================================================================== --- llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h +++ llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h @@ -177,7 +177,7 @@ mutable Optional NeedsDwarfUnwindInfo; /// True if the function need asynchronous unwind information. - mutable Optional NeedsDwarfAsyncUnwindInfo; + mutable Optional NeedsAsyncDwarfUnwindInfo; public: explicit AArch64FunctionInfo(MachineFunction &MF); Index: llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp +++ llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp @@ -126,9 +126,19 @@ } bool AArch64FunctionInfo::needsAsyncDwarfUnwindInfo() const { - if (!NeedsDwarfAsyncUnwindInfo.hasValue()) - NeedsDwarfAsyncUnwindInfo = - needsDwarfUnwindInfo() && - MF.getFunction().getUWTableKind() == UWTableKind::Async; - return NeedsDwarfAsyncUnwindInfo.getValue(); + if (!NeedsAsyncDwarfUnwindInfo.hasValue()) { + const Function &F = MF.getFunction(); + NeedsAsyncDwarfUnwindInfo = + needsDwarfUnwindInfo() && F.getUWTableKind() == UWTableKind::Async && + !MF.getSubtarget() + .isTargetMachO() && // TODO: async unwind info not represenatble in + // the compact format(?). + !F.hasMinSize(); // TODO: this is to prevent epilogue unwind info + // from being emitted for homogeneous epilogues, + // outlined functions, and functions outlined from. + // Alternatively, we could disable those + // optimisations. Or even better, add async unwind + // support to them! + } + return NeedsAsyncDwarfUnwindInfo.getValue(); } Index: llvm/test/CodeGen/AArch64/GlobalISel/byval-call.ll =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/byval-call.ll +++ llvm/test/CodeGen/AArch64/GlobalISel/byval-call.ll @@ -3,7 +3,7 @@ declare void @byval_i32(i32* byval(i32) %ptr) -define void @call_byval_i32(i32* %incoming) { +define void @call_byval_i32(i32* %incoming) uwtable { ; CHECK-LABEL: call_byval_i32: ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #32 @@ -15,6 +15,8 @@ ; CHECK-NEXT: bl byval_i32 ; CHECK-NEXT: ldr x30, [sp, #16] // 8-byte Folded Reload ; CHECK-NEXT: add sp, sp, #32 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret call void @byval_i32(i32* byval(i32) %incoming) ret void @@ -22,7 +24,7 @@ declare void @byval_a64i32([64 x i32]* byval([64 x i32]) %ptr) -define void @call_byval_a64i32([64 x i32]* %incoming) { +define void @call_byval_a64i32([64 x i32]* %incoming) uwtable { ; CHECK-LABEL: call_byval_a64i32: ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #288 @@ -67,9 +69,14 @@ ; CHECK-NEXT: ldr q0, [x0, #240] ; CHECK-NEXT: str q0, [sp, #240] ; CHECK-NEXT: bl byval_a64i32 +; CHECK-NEXT: .cfi_def_cfa wsp, 288 ; CHECK-NEXT: ldr x28, [sp, #272] // 8-byte Folded Reload ; CHECK-NEXT: ldp x29, x30, [sp, #256] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #288 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w28 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret call void @byval_a64i32([64 x i32]* byval([64 x i32]) %incoming) ret void Index: llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll =================================================================== --- llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll +++ llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll @@ -444,6 +444,12 @@ ; CHECK-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: ldr x30, [sp], #48 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w19 +; CHECK-NEXT: .cfi_restore w20 +; CHECK-NEXT: .cfi_restore w21 +; CHECK-NEXT: .cfi_restore w22 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret entry: %size = getelementptr inbounds %struct.Struct, %struct.Struct* %hdCall, i64 0, i32 0 @@ -513,6 +519,10 @@ ; CHECK-NEXT: .LBB7_8: // %return ; CHECK-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: ldr x30, [sp], #32 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w19 +; CHECK-NEXT: .cfi_restore w20 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret entry: %0 = load i32, i32* @a, align 4 @@ -589,10 +599,16 @@ ; CHECK-NEXT: // %bb.5: ; CHECK-NEXT: mov w0, #123 ; CHECK-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w19 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret ; CHECK-NEXT: .LBB8_6: // %if.end ; CHECK-NEXT: mov w0, wzr ; CHECK-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w19 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret entry: %0 = load i32, i32* @a, align 4 @@ -638,7 +654,7 @@ ; fcmp d8, #0.0 ; b.gt .LBB0_5 -define i32 @fcmpri(i32 %argc, i8** nocapture readonly %argv) { +define i32 @fcmpri(i32 %argc, i8** nocapture readonly %argv) #0 { ; CHECK-LABEL: fcmpri: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: str d8, [sp, #-32]! // 8-byte Folded Spill @@ -676,6 +692,10 @@ ; CHECK-NEXT: .LBB9_4: // %return ; CHECK-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: ldr d8, [sp], #32 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w19 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore b8 ; CHECK-NEXT: ret ; CHECK-LABEL-DAG: .LBB9_3 @@ -714,7 +734,7 @@ ret i32 %retval.0 } -define void @cmp_shifted(i32 %in, i32 %lhs, i32 %rhs) { +define void @cmp_shifted(i32 %in, i32 %lhs, i32 %rhs) #0 { ; CHECK-LABEL: cmp_shifted: ; CHECK: // %bb.0: // %common.ret ; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill @@ -728,6 +748,8 @@ ; CHECK-NEXT: csel w0, w9, w8, ge ; CHECK-NEXT: bl zoo ; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret ; [...] @@ -830,3 +852,5 @@ declare i32 @xoo(i32, i32) declare i32 @woo(double, double) + +attributes #0 = { uwtable } \ No newline at end of file Index: llvm/test/CodeGen/AArch64/cond-br-tuning.ll =================================================================== --- llvm/test/CodeGen/AArch64/cond-br-tuning.ll +++ llvm/test/CodeGen/AArch64/cond-br-tuning.ll @@ -180,7 +180,7 @@ declare void @bar(i32) ; Don't transform since the call will clobber the NZCV bits. -define void @test_call_clobber(i32 %unused, i32 %a) { +define void @test_call_clobber(i32 %unused, i32 %a) uwtable { ; CHECK-LABEL: test_call_clobber: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill @@ -193,6 +193,9 @@ ; CHECK-NEXT: cbnz w19, .LBB9_2 ; CHECK-NEXT: // %bb.1: // %if.end ; CHECK-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w19 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret ; CHECK-NEXT: .LBB9_2: // %if.then ; CHECK-NEXT: bl foo Index: llvm/test/CodeGen/AArch64/csr-split.ll =================================================================== --- llvm/test/CodeGen/AArch64/csr-split.ll +++ llvm/test/CodeGen/AArch64/csr-split.ll @@ -6,7 +6,7 @@ @a = common dso_local local_unnamed_addr global i32 0, align 4 -define dso_local signext i32 @test1(i32* %b) local_unnamed_addr { +define dso_local signext i32 @test1(i32* %b) local_unnamed_addr uwtable { ; CHECK-LABEL: test1: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill @@ -19,12 +19,18 @@ ; CHECK-NEXT: b.eq .LBB0_2 ; CHECK-NEXT: // %bb.1: // %if.end ; CHECK-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w19 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret ; CHECK-NEXT: .LBB0_2: // %if.then ; CHECK-NEXT: mov x19, x0 ; CHECK-NEXT: bl callVoid ; CHECK-NEXT: mov x0, x19 ; CHECK-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w19 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: b callNonVoid ; ; CHECK-APPLE-LABEL: test1: @@ -75,7 +81,7 @@ declare signext i32 @callNonVoid(i32*) local_unnamed_addr -define dso_local signext i32 @test2(i32* %p1) local_unnamed_addr { +define dso_local signext i32 @test2(i32* %p1) local_unnamed_addr uwtable { ; CHECK-LABEL: test2: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill @@ -93,10 +99,16 @@ ; CHECK-NEXT: bl callVoid ; CHECK-NEXT: mov x0, x19 ; CHECK-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w19 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: b callNonVoid ; CHECK-NEXT: .LBB1_3: // %return ; CHECK-NEXT: mov w0, wzr ; CHECK-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w19 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret ; ; CHECK-APPLE-LABEL: test2: @@ -151,7 +163,7 @@ } -define dso_local i8* @test3(i8** nocapture %p1, i8 zeroext %p2) local_unnamed_addr { +define dso_local i8* @test3(i8** nocapture %p1, i8 zeroext %p2) local_unnamed_addr uwtable { ; CHECK-LABEL: test3: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: str x30, [sp, #-32]! // 8-byte Folded Spill @@ -171,6 +183,10 @@ ; CHECK-NEXT: mov x0, x19 ; CHECK-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: ldr x30, [sp], #32 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w19 +; CHECK-NEXT: .cfi_restore w20 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret ; ; CHECK-APPLE-LABEL: test3: Index: llvm/test/CodeGen/AArch64/emutls.ll =================================================================== --- llvm/test/CodeGen/AArch64/emutls.ll +++ llvm/test/CodeGen/AArch64/emutls.ll @@ -9,12 +9,13 @@ @my_emutls_v_xyz = external global i8*, align 4 declare i8* @my_emutls_get_address(i8*) -define i32 @my_get_xyz() { +define i32 @my_get_xyz() uwtable { ; ARM64-LABEL: my_get_xyz: ; ARM64: adrp x0, :got:my_emutls_v_xyz ; ARM64-NEXT: ldr x0, [x0, :got_lo12:my_emutls_v_xyz] ; ARM64-NEXT: bl my_emutls_get_address ; ARM64-NEXT: ldr w0, [x0] +; ARM64-NEXT: .cfi_def_cfa wsp, 16 ; ARM64-NEXT: ldp x29, x30, [sp] entry: @@ -32,12 +33,13 @@ @s1 = thread_local global i16 15 @b1 = thread_local global i8 0 -define i32 @f1() { +define i32 @f1() uwtable { ; ARM64-LABEL: f1: ; ARM64: adrp x0, :got:__emutls_v.i1 ; ARM64-NEXT: ldr x0, [x0, :got_lo12:__emutls_v.i1] ; ARM64-NEXT: bl __emutls_get_address ; ARM64-NEXT: ldr w0, [x0] +; ARM64-NEXT: .cfi_def_cfa wsp, 16 ; ARM64-NEXT: ldp x29, x30, [sp] entry: @@ -45,11 +47,12 @@ ret i32 %tmp1 } -define i32* @f2() { +define i32* @f2() uwtable { ; ARM64-LABEL: f2: ; ARM64: adrp x0, :got:__emutls_v.i1 ; ARM64-NEXT: ldr x0, [x0, :got_lo12:__emutls_v.i1] ; ARM64-NEXT: bl __emutls_get_address +; ARM64-NEXT: .cfi_def_cfa wsp, 16 ; ARM64-NEXT: ldp x29, x30, [sp] entry: @@ -68,11 +71,12 @@ ret i32 %tmp1 } -define i32* @f6() { +define i32* @f6() uwtable { ; ARM64-LABEL: f6: ; ARM64: adrp x0, __emutls_v.i3 ; ARM64: add x0, x0, :lo12:__emutls_v.i3 ; ARM64-NEXT: bl __emutls_get_address +; ARM64-NEXT: .cfi_def_cfa wsp, 16 ; ARM64-NEXT: ldp x29, x30, [sp] entry: Index: llvm/test/CodeGen/AArch64/fastcc.ll =================================================================== --- llvm/test/CodeGen/AArch64/fastcc.ll +++ llvm/test/CodeGen/AArch64/fastcc.ll @@ -5,7 +5,7 @@ ; Without tailcallopt fastcc still means the caller cleans up the ; stack, so try to make sure this is respected. -define fastcc void @func_stack0() { +define fastcc void @func_stack0() uwtable { ; CHECK-LABEL: func_stack0: ; CHECK: sub sp, sp, #48 ; CHECK: add x29, sp, #32 @@ -49,15 +49,21 @@ ret void ; CHECK: ldp x29, x30, [sp, #32] ; CHECK-NEXT: add sp, sp, #48 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret ; CHECK-TAIL: ldp x29, x30, [sp, #32] ; CHECK-TAIL-NEXT: add sp, sp, #48 +; CHECK-TAIL-NEXT: .cfi_def_cfa_offset 0 +; CHECK-TAIL-NEXT: .cfi_restore w30 +; CHECK-TAIL-NEXT: .cfi_restore w29 ; CHECK-TAIL-NEXT: ret } -define fastcc void @func_stack8([8 x i64], i32 %stacked) { +define fastcc void @func_stack8([8 x i64], i32 %stacked) uwtable { ; CHECK-LABEL: func_stack8: ; CHECK: sub sp, sp, #48 ; CHECK: stp x29, x30, [sp, #32] @@ -100,17 +106,24 @@ ; CHECK-TAIL-NOT: sub sp, sp ret void +; CHECK-NEXT: .cfi_def_cfa wsp, 48 ; CHECK-NEXT: ldp x29, x30, [sp, #32] -; CHECK: add sp, sp, #48 +; CHECK-NEXT: add sp, sp, #48 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret ; CHECK-TAIL: ldp x29, x30, [sp, #32] ; CHECK-TAIL-NEXT: add sp, sp, #64 +; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -16 +; CHECK-TAIL-NEXT: .cfi_restore w30 +; CHECK-TAIL-NEXT: .cfi_restore w29 ; CHECK-TAIL-NEXT: ret } -define fastcc void @func_stack32([8 x i64], i128 %stacked0, i128 %stacked1) { +define fastcc void @func_stack32([8 x i64], i128 %stacked0, i128 %stacked1) uwtable { ; CHECK-LABEL: func_stack32: ; CHECK: add x29, sp, #32 @@ -146,22 +159,31 @@ ; CHECK-TAIL-NOT: sub sp, sp ret void -; CHECK: ldp x29, x30, [sp, #32] +; CHECK: .cfi_def_cfa wsp, 48 +; CHECK-NEXT: ldp x29, x30, [sp, #32] ; CHECK-NEXT: add sp, sp, #48 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret ; CHECK-TAIL: ldp x29, x30, [sp, #32] ; CHECK-TAIL-NEXT: add sp, sp, #80 +; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -32 +; CHECK-TAIL-NEXT: .cfi_restore w30 +; CHECK-TAIL-NEXT: .cfi_restore w29 ; CHECK-TAIL-NEXT: ret } ; Check that arg stack pop is done after callee-save restore when no frame pointer is used. -define fastcc void @func_stack32_leaf([8 x i64], i128 %stacked0, i128 %stacked1) { +define fastcc void @func_stack32_leaf([8 x i64], i128 %stacked0, i128 %stacked1) uwtable { ; CHECK-LABEL: func_stack32_leaf: ; CHECK: str x20, [sp, #-16]! ; CHECK: nop ; CHECK-NEXT: //NO_APP ; CHECK-NEXT: ldr x20, [sp], #16 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w20 ; CHECK-NEXT: ret ; CHECK-TAIL-LABEL: func_stack32_leaf: @@ -169,7 +191,10 @@ ; CHECK-TAIL: nop ; CHECK-TAIL-NEXT: //NO_APP ; CHECK-TAIL-NEXT: ldr x20, [sp], #16 -; CHECK-TAIL-NEXT: add sp, sp, #32 +; CHECK-TAIL-NEXT: .cfi_def_cfa_offset 0 +; CHECK-TAIL-NEXT: add sp, sp, #32 +; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -32 +; CHECK-TAIL-NEXT: .cfi_restore w20 ; CHECK-TAIL-NEXT: ret ; CHECK-TAIL-RZ-LABEL: func_stack32_leaf: @@ -178,7 +203,10 @@ ; CHECK-TAIL-RZ: nop ; CHECK-TAIL-RZ-NEXT: //NO_APP ; CHECK-TAIL-RZ-NEXT: ldr x20, [sp], #16 -; CHECK-TAIL-RZ-NEXT: add sp, sp, #32 +; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset 0 +; CHECK-TAIL-RZ-NEXT: add sp, sp, #32 +; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset -32 +; CHECK-TAIL-RZ-NEXT: .cfi_restore w20 ; CHECK-TAIL-RZ-NEXT: ret ; Make sure there is a callee-save register to save/restore. @@ -187,7 +215,7 @@ } ; Check that arg stack pop is done after callee-save restore when no frame pointer is used. -define fastcc void @func_stack32_leaf_local([8 x i64], i128 %stacked0, i128 %stacked1) { +define fastcc void @func_stack32_leaf_local([8 x i64], i128 %stacked0, i128 %stacked1) uwtable { ; CHECK-LABEL: func_stack32_leaf_local: ; CHECK: sub sp, sp, #32 ; CHECK-NEXT: .cfi_def_cfa_offset 32 @@ -196,6 +224,8 @@ ; CHECK-NEXT: //NO_APP ; CHECK-NEXT: ldr x20, [sp, #16] ; CHECK-NEXT: add sp, sp, #32 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w20 ; CHECK-NEXT: ret ; CHECK-TAIL-LABEL: func_stack32_leaf_local: @@ -206,6 +236,8 @@ ; CHECK-TAIL-NEXT: //NO_APP ; CHECK-TAIL-NEXT: ldr x20, [sp, #16] ; CHECK-TAIL-NEXT: add sp, sp, #64 +; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -32 +; CHECK-TAIL-NEXT: .cfi_restore w20 ; CHECK-TAIL-NEXT: ret ; CHECK-TAIL-RZ-LABEL: func_stack32_leaf_local: @@ -214,7 +246,10 @@ ; CHECK-TAIL-RZ: nop ; CHECK-TAIL-RZ-NEXT: //NO_APP ; CHECK-TAIL-RZ-NEXT: ldr x20, [sp], #16 -; CHECK-TAIL-RZ-NEXT: add sp, sp, #32 +; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset 0 +; CHECK-TAIL-RZ-NEXT: add sp, sp, #32 +; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset -32 +; CHECK-TAIL-RZ-NEXT: .cfi_restore w20 ; CHECK-TAIL-RZ-NEXT: ret %val0 = alloca [2 x i64], align 8 @@ -225,19 +260,24 @@ } ; Check that arg stack pop is done after callee-save restore when no frame pointer is used. -define fastcc void @func_stack32_leaf_local_nocs([8 x i64], i128 %stacked0, i128 %stacked1) { +define fastcc void @func_stack32_leaf_local_nocs([8 x i64], i128 %stacked0, i128 %stacked1) uwtable { ; CHECK-LABEL: func_stack32_leaf_local_nocs: ; CHECK: sub sp, sp, #16 -; CHECK: add sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: add sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 0 ; CHECK-NEXT: ret ; CHECK-TAIL-LABEL: func_stack32_leaf_local_nocs: ; CHECK-TAIL: sub sp, sp, #16 -; CHECK-TAIL: add sp, sp, #48 +; CHECK-TAIL-NEXT: .cfi_def_cfa_offset 16 +; CHECK-TAIL-NEXT: add sp, sp, #48 +; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -32 ; CHECK-TAIL-NEXT: ret ; CHECK-TAIL-RZ-LABEL: func_stack32_leaf_local_nocs: ; CHECK-TAIL-RZ: add sp, sp, #32 +; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset -32 ; CHECK-TAIL-RZ-NEXT: ret %val0 = alloca [2 x i64], align 8 Index: llvm/test/CodeGen/AArch64/framelayout-sve-calleesaves-fix.mir =================================================================== --- llvm/test/CodeGen/AArch64/framelayout-sve-calleesaves-fix.mir +++ llvm/test/CodeGen/AArch64/framelayout-sve-calleesaves-fix.mir @@ -2,7 +2,7 @@ # RUN: llc -mattr=+sve -mtriple=aarch64-none-linux-gnu -start-before=prologepilog %s -o - | FileCheck %s --- | - define aarch64_sve_vector_pcs void @fix_restorepoint_p4() { entry: unreachable } + define aarch64_sve_vector_pcs void @fix_restorepoint_p4() uwtable { entry: unreachable } ; CHECK-LABEL: fix_restorepoint_p4: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill @@ -18,10 +18,15 @@ ; CHECK-NEXT: // implicit-def: $z8 ; CHECK-NEXT: // implicit-def: $p4 ; CHECK-NEXT: addvl sp, sp, #1 + ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #2 + ; CHECK-NEXT: .cfi_def_cfa wsp, 16 + ; CHECK-NEXT: .cfi_restore z8 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload + ; CHECK-NEXT: .cfi_def_cfa_offset 0 + ; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret ... name: fix_restorepoint_p4 Index: llvm/test/CodeGen/AArch64/framelayout-sve.mir =================================================================== --- llvm/test/CodeGen/AArch64/framelayout-sve.mir +++ llvm/test/CodeGen/AArch64/framelayout-sve.mir @@ -27,20 +27,20 @@ # --- | - define void @test_allocate_sve() { entry: unreachable } - define void @test_allocate_sve_gpr_callee_saves() { entry: unreachable } - define void @test_allocate_sve_gpr_realigned() { entry: unreachable } - define void @test_address_sve() { entry: unreachable } - define void @test_address_sve_fp() { entry: unreachable } - define void @test_stack_arg_sve() { entry: unreachable } - define void @test_address_sve_out_of_range() { entry: unreachable } - define void @test_address_gpr_vla() { entry: unreachable } - define aarch64_sve_vector_pcs void @save_restore_pregs_sve() { entry: unreachable } - define aarch64_sve_vector_pcs void @save_restore_zregs_sve() { entry: unreachable } - define aarch64_sve_vector_pcs void @save_restore_sve() { entry: unreachable } - define aarch64_sve_vector_pcs void @save_restore_sve_realign() { entry: unreachable } - define aarch64_sve_vector_pcs void @frame_layout() { entry: unreachable } - define void @fp_relative_index_with_float_save() { entry: unreachable } + define void @test_allocate_sve() uwtable { entry: unreachable } + define void @test_allocate_sve_gpr_callee_saves() uwtable { entry: unreachable } + define void @test_allocate_sve_gpr_realigned() uwtable { entry: unreachable } + define void @test_address_sve() uwtable { entry: unreachable } + define void @test_address_sve_fp() uwtable { entry: unreachable } + define void @test_stack_arg_sve() uwtable { entry: unreachable } + define void @test_address_sve_out_of_range() uwtable { entry: unreachable } + define void @test_address_gpr_vla() uwtable { entry: unreachable } + define aarch64_sve_vector_pcs void @save_restore_pregs_sve() uwtable { entry: unreachable } + define aarch64_sve_vector_pcs void @save_restore_zregs_sve() uwtable { entry: unreachable } + define aarch64_sve_vector_pcs void @save_restore_sve() uwtable { entry: unreachable } + define aarch64_sve_vector_pcs void @save_restore_sve_realign() uwtable { entry: unreachable } + define aarch64_sve_vector_pcs void @frame_layout() uwtable { entry: unreachable } + define void @fp_relative_index_with_float_save() uwtable { entry: unreachable } ... # +----------+ @@ -65,8 +65,12 @@ # CHECK-NEXT: frame-setup CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 2 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: RET_ReallyLR # ASM-LABEL: test_allocate_sve: @@ -74,11 +78,21 @@ # ASM-NEXT: .cfi_offset w29, -16 # ASM: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG # ASM: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 16 * VG +# ASM: .cfi_def_cfa wsp, 32 +# ASM: .cfi_def_cfa_offset 16 +# ASM: .cfi_def_cfa_offset 0 +# ASM-NEXT: .cfi_restore w29 + # # UNWINDINFO: DW_CFA_def_cfa_offset: +16 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16 # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# UNWINDINFO: DW_CFA_def_cfa: reg31 +32 +# UNWINDINFO: DW_CFA_def_cfa_offset: +16 +# UNWINDINFO: DW_CFA_def_cfa_offset: +0 +# UNWINDINFO-NEXT: DW_CFA_restore: reg29 + name: test_allocate_sve stack: @@ -114,9 +128,12 @@ # CHECK-NEXT: $x20 = IMPLICIT_DEF # CHECK-NEXT: $x21 = IMPLICIT_DEF # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 2 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $x21, $x20 = frame-destroy LDPXi $sp, 2 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 32 +# CHECK-COUNT-4: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: RET_ReallyLR # # ASM-LABEL: test_allocate_sve_gpr_callee_saves: @@ -126,6 +143,14 @@ # ASM-NEXT: .cfi_offset w29, -32 # ASM: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 16 * VG # ASM: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 16 * VG +# +# ASM: .cfi_def_cfa wsp, 48 +# ASM: .cfi_def_cfa_offset 32 +# ASM: .cfi_def_cfa_offset 0 +# ASM-NEXT: .cfi_restore w20 +# ASM-NEXT: .cfi_restore w21 +# ASM-NEXT: .cfi_restore w29 + # # UNWINDINFO: DW_CFA_def_cfa_offset: +32 # UNWINDINFO: DW_CFA_offset: reg20 -8 @@ -133,6 +158,14 @@ # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -32 # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# +# UNWINDINFO: DW_CFA_def_cfa: reg31 +48 +# UNWINDINFO: DW_CFA_def_cfa_offset: +32 +# UNWINDINFO: DW_CFA_def_cfa_offset: +0 +# UNWINDINFO-NEXT: DW_CFA_restore: reg20 +# UNWINDINFO-NEXT: DW_CFA_restore: reg21 +# UNWINDINFO-NEXT: DW_CFA_restore: reg29 + name: test_allocate_sve_gpr_callee_saves stack: - { id: 0, stack-id: scalable-vector, size: 18, alignment: 2 } @@ -164,7 +197,9 @@ # CHECK-NEXT: $[[TMP:x[0-9]+]] = frame-setup SUBXri $sp, 16, 0 # CHECK-NEXT: $sp = ANDXri killed $[[TMP]] # CHECK-NEXT: $sp = frame-destroy ADDXri $fp, 0, 0 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2 +# CHECK-COUNT-3: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: RET_ReallyLR # # ASM-LABEL: test_allocate_sve_gpr_realigned: @@ -172,11 +207,23 @@ # ASM: .cfi_def_cfa w29, 16 # ASM-NEXT: .cfi_offset w30, -8 # ASM-NEXT: .cfi_offset w29, -16 +# +# ASM: .cfi_def_cfa wsp, 16 +# ASM: .cfi_def_cfa_offset 0 +# ASM-NEXT: .cfi_restore w30 +# ASM-NEXT: .cfi_restore w29 + # # UNWINDINFO: DW_CFA_def_cfa_offset: +16 # UNWINDINFO: DW_CFA_def_cfa: reg29 +16 # UNWINDINFO-NEXT: DW_CFA_offset: reg30 -8 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16 +# +# UNWINDINFO: DW_CFA_def_cfa: reg31 +16 +# UNWINDINFO: DW_CFA_def_cfa_offset: +0 +# UNWINDINFO-NEXT: DW_CFA_restore: reg30 +# UNWINDINFO-NEXT: DW_CFA_restore: reg29 + name: test_allocate_sve_gpr_realigned stack: @@ -216,8 +263,12 @@ # CHECK-NEXT: STR_PXI $p0, killed $[[TMP]], 7 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 3 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: RET_ReallyLR # # ASM-LABEL: test_address_sve: @@ -226,10 +277,21 @@ # ASM: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG # ASM: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 24 * VG # +# ASM: .cfi_def_cfa wsp, 32 +# ASM: .cfi_def_cfa_offset 16 +# ASM: .cfi_def_cfa_offset 0 +# ASM-NEXT: .cfi_restore w29 +# # UNWINDINFO: DW_CFA_def_cfa_offset: +16 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16 # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# +# UNWINDINFO: DW_CFA_def_cfa: reg31 +32 +# UNWINDINFO: DW_CFA_def_cfa_offset: +16 +# UNWINDINFO: DW_CFA_def_cfa_offset: +0 +# UNWINDINFO-NEXT: DW_CFA_restore: reg29 + name: test_address_sve frameInfo: @@ -278,7 +340,9 @@ # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 3 # CHECK: $sp = frame-destroy ADDXri $sp, 16, 0 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2 +# CHECK-COUNT-3: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: RET_ReallyLR # # ASM-LABEL: test_address_sve_fp: @@ -287,10 +351,21 @@ # ASM-NEXT: .cfi_offset w30, -8 # ASM-NEXT: .cfi_offset w29, -16 # +# ASM: .cfi_def_cfa wsp, 16 +# ASM: .cfi_def_cfa_offset 0 +# ASM-NEXT: .cfi_restore w30 +# ASM-NEXT: .cfi_restore w29 +# # UNWINDINFO: DW_CFA_def_cfa_offset: +16 # UNWINDINFO: DW_CFA_def_cfa: reg29 +16 # UNWINDINFO-NEXT: DW_CFA_offset: reg30 -8 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16 +# +# UNWINDINFO: DW_CFA_def_cfa: reg31 +16 +# UNWINDINFO: DW_CFA_def_cfa_offset: +0 +# UNWINDINFO-NEXT: DW_CFA_restore: reg30 +# UNWINDINFO-NEXT: DW_CFA_restore: reg29 + name: test_address_sve_fp frameInfo: maxAlignment: 16 @@ -334,8 +409,12 @@ # CHECK-NEXT: $x0 = LDRXui killed $[[TMP]], 4 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 1 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: RET_ReallyLR # # ASM-LABEL: test_stack_arg_sve: @@ -344,10 +423,20 @@ # ASM: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG # ASM: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 8 * VG # +# ASM: .cfi_def_cfa wsp, 32 +# ASM: .cfi_def_cfa_offset 16 +# ASM: .cfi_def_cfa_offset 0 +# ASM-NEXT: .cfi_restore w29 + # UNWINDINFO: DW_CFA_def_cfa_offset: +16 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16 # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# +# UNWINDINFO: DW_CFA_def_cfa: reg31 +32 +# UNWINDINFO: DW_CFA_def_cfa_offset: +16 +# UNWINDINFO: DW_CFA_def_cfa_offset: +0 +# UNWINDINFO-NEXT: DW_CFA_restore: reg29 name: test_stack_arg_sve fixedStack: @@ -407,15 +496,26 @@ # CHECK-NEXT: STR_PXI $p0, killed $[[TMP2]], 255 # CHECK: $sp = frame-destroy ADDVL_XXI $sp, 31 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 9 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: RET_ReallyLR # # ASM-LABEL: test_address_sve_out_of_range: @@ -431,6 +531,18 @@ # ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 2048 * VG # ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x88, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 2056 * VG # +# ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x0e, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1808 * VG +# ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x98, 0x0c, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1560 * VG +# ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xa0, 0x0a, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1312 * VG +# ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xa8, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1064 * VG +# ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xb0, 0x06, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 816 * VG +# ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xb8, 0x04, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 568 * VG +# ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc0, 0x02, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 320 * VG +# ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc8, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 72 * VG +# ASM: .cfi_def_cfa wsp, 16 +# ASM: .cfi_def_cfa_offset 0 +# ASM-NEXT: .cfi_restore w29 + # UNWINDINFO: DW_CFA_def_cfa_offset: +16 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16 # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +256, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus @@ -442,6 +554,19 @@ # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1792, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +2048, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +2056, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# +# UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1808, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1560, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1312, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1064, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +816, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +568, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +320, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +72, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# UNWINDINFO: DW_CFA_def_cfa: reg31 +16 +# UNWINDINFO: DW_CFA_def_cfa_offset: +0 +# UNWINDINFO-NEXT: DW_CFA_restore: reg29 + name: test_address_sve_out_of_range frameInfo: maxAlignment: 16 @@ -484,11 +609,24 @@ # ASM-NEXT: .cfi_offset w30, -24 # ASM-NEXT: .cfi_offset w29, -32 # +# ASM: .cfi_def_cfa wsp, 32 +# ASM: .cfi_def_cfa_offset 0 +# ASM-NEXT: .cfi_restore w19 +# ASM-NEXT: .cfi_restore w30 +# ASM-NEXT: .cfi_restore w29 + # UNWINDINFO: DW_CFA_def_cfa_offset: +32 # UNWINDINFO: DW_CFA_def_cfa: reg29 +32 # UNWINDINFO-NEXT: DW_CFA_offset: reg19 -16 # UNWINDINFO-NEXT: DW_CFA_offset: reg30 -24 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -32 +# +# UNWINDINFO: DW_CFA_def_cfa: reg31 +32 +# UNWINDINFO: DW_CFA_def_cfa_offset: +0 +# UNWINDINFO-NEXT: DW_CFA_restore: reg19 +# UNWINDINFO-NEXT: DW_CFA_restore: reg30 +# UNWINDINFO-NEXT: DW_CFA_restore: reg29 + name: test_address_gpr_vla frameInfo: maxAlignment: 16 @@ -528,10 +666,21 @@ # ASM: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG # ASM: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 8 * VG # +# ASM: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG +# ASM: .cfi_def_cfa wsp, 16 +# ASM: .cfi_def_cfa_offset 0 +# ASM-NEXT: .cfi_restore w29 + # UNWINDINFO: DW_CFA_def_cfa_offset: +16 # UNWINDINFO: DW_CFA_offset: reg29 -16 # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# +# UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# UNWINDINFO: DW_CFA_def_cfa: reg31 +16 +# UNWINDINFO: DW_CFA_def_cfa_offset: +0 +# UNWINDINFO-NEXT: DW_CFA_restore: reg29 + name: save_restore_pregs_sve stack: - { id: 0, stack-id: default, size: 32, alignment: 16 } @@ -558,11 +707,15 @@ # CHECK-NEXT: frame-setup CFI_INSTRUCTION # CHECK: $sp = frame-destroy ADDXri $sp, 32, 0 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $z10 = frame-destroy LDR_ZXI $sp, 0 # CHECK-NEXT: $z9 = frame-destroy LDR_ZXI $sp, 1 # CHECK-NEXT: $z8 = frame-destroy LDR_ZXI $sp, 2 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 3 +# CHECK-COUNT-4: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp, $fp = frame-destroy LDRXpost $sp, 16 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: RET_ReallyLR # # ASM-LABEL: save_restore_zregs_sve: @@ -573,6 +726,14 @@ # ASM-NEXT: .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG # ASM-NEXT: .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG # ASM: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 24 * VG +# +# ASM: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG +# ASM: .cfi_def_cfa wsp, 16 +# ASM-NEXT: .cfi_restore z8 +# ASM-NEXT: .cfi_restore z9 +# ASM-NEXT: .cfi_restore z10 +# ASM: .cfi_def_cfa_offset 0 +# ASM-NEXT: .cfi_restore w29 # UNWINDINFO: DW_CFA_def_cfa_offset: +16 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16 @@ -581,6 +742,14 @@ # UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# +# UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# UNWINDINFO: DW_CFA_def_cfa: reg31 +16 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg104 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg105 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg106 +# UNWINDINFO: DW_CFA_def_cfa_offset: +0 +# UNWINDINFO-NEXT: DW_CFA_restore: reg29 name: save_restore_zregs_sve stack: @@ -623,7 +792,9 @@ # CHECK: frame-setup CFI_INSTRUCTION # CHECK: $sp = frame-destroy ADDXri $sp, 32, 0 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK: $sp = frame-destroy ADDVL_XXI $sp, 1 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK: $p15 = frame-destroy LDR_PXI $sp, 4 # CHECK: $p14 = frame-destroy LDR_PXI $sp, 5 # CHECK: $p5 = frame-destroy LDR_PXI $sp, 14 @@ -633,8 +804,10 @@ # CHECK: $z9 = frame-destroy LDR_ZXI $sp, 16 # CHECK: $z8 = frame-destroy LDR_ZXI $sp, 17 # CHECK: $sp = frame-destroy ADDVL_XXI $sp, 18 +# CHECK-COUNT-9: frame-destroy CFI_INSTRUCTION # CHECK: $x20, $x19 = frame-destroy LDPXi $sp, 2 # CHECK: $sp, ${{[a-z0-9]+}}, $x21 = frame-destroy LDPXpost $sp, 4 +# CHECK-COUNT-5: frame-destroy CFI_INSTRUCTION # CHECK: RET_ReallyLR # # ASM-LABEL: save_restore_sve: @@ -643,7 +816,7 @@ # ASM-NEXT: .cfi_offset w20, -16 # ASM-NEXT: .cfi_offset w21, -24 # ASM-NEXT: .cfi_offset w29, -32 -# ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 144 * VG +# ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 144 * VG # ASM: .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 32 - 8 * VG # ASM-NEXT: .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 32 - 16 * VG # ASM-NEXT: .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 32 - 24 * VG @@ -655,6 +828,23 @@ # ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x98, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 152 * VG # ASM: .cfi_escape 0x0f, 0x0e, 0x8f, 0x00, 0x11, 0xc0, 0x00, 0x22, 0x11, 0x98, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 64 + 152 * VG # +# ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x98, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 152 * VG +# ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 144 * VG +# ASM: .cfi_def_cfa wsp, 32 +# ASM-NEXT: .cfi_restore z8 +# ASM-NEXT: .cfi_restore z9 +# ASM-NEXT: .cfi_restore z10 +# ASM-NEXT: .cfi_restore z11 +# ASM-NEXT: .cfi_restore z12 +# ASM-NEXT: .cfi_restore z13 +# ASM-NEXT: .cfi_restore z14 +# ASM-NEXT: .cfi_restore z15 +# ASM: .cfi_def_cfa_offset 0 +# ASM-NEXT: .cfi_restore w19 +# ASM-NEXT: .cfi_restore w20 +# ASM-NEXT: .cfi_restore w21 +# ASM-NEXT: .cfi_restore w29 + # UNWINDINFO: DW_CFA_def_cfa_offset: +32 # UNWINDINFO: DW_CFA_offset: reg19 -8 # UNWINDINFO-NEXT: DW_CFA_offset: reg20 -16 @@ -671,6 +861,23 @@ # UNWINDINFO-NEXT: DW_CFA_expression: reg79 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -64, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +152, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +64, DW_OP_plus, DW_OP_consts +152, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# +# UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +152, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +144, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# UNWINDINFO: DW_CFA_def_cfa: reg31 +32 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg104 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg105 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg106 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg107 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg108 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg109 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg110 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg111 +# UNWINDINFO: DW_CFA_def_cfa_offset: +0 +# UNWINDINFO-NEXT: DW_CFA_restore: reg19 +# UNWINDINFO-NEXT: DW_CFA_restore: reg20 +# UNWINDINFO-NEXT: DW_CFA_restore: reg21 +# UNWINDINFO-NEXT: DW_CFA_restore: reg29 name: save_restore_sve stack: @@ -737,8 +944,11 @@ # CHECK-NEXT: $z22 = frame-destroy LDR_ZXI $sp, 3 # CHECK: $z9 = frame-destroy LDR_ZXI $sp, 16 # CHECK-NEXT: $z8 = frame-destroy LDR_ZXI $sp, 17 +# CHECK-COUNT-8: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-destroy ADDXri $fp, 0, 0 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2 +# CHECK-COUNT-3: frame-destroy CFI_INSTRUCTION # CHECK-NEXT: RET_ReallyLR # # ASM-LABEL: save_restore_sve_realign: @@ -755,6 +965,19 @@ # ASM-NEXT: .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 16 - 56 * VG # ASM-NEXT: .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 16 - 64 * VG # +# ASM: .cfi_restore z8 +# ASM-NEXT: .cfi_restore z9 +# ASM-NEXT: .cfi_restore z10 +# ASM-NEXT: .cfi_restore z11 +# ASM-NEXT: .cfi_restore z12 +# ASM-NEXT: .cfi_restore z13 +# ASM-NEXT: .cfi_restore z14 +# ASM-NEXT: .cfi_restore z15 +# ASM: .cfi_def_cfa wsp, 16 +# ASM: .cfi_def_cfa_offset 0 +# ASM-NEXT: .cfi_restore w30 +# ASM-NEXT: .cfi_restore w29 +# # UNWINDINFO: DW_CFA_def_cfa_offset: +16 # UNWINDINFO: DW_CFA_def_cfa: reg29 +16 # UNWINDINFO-NEXT: DW_CFA_offset: reg30 -8 @@ -767,6 +990,20 @@ # UNWINDINFO-NEXT: DW_CFA_expression: reg77 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -48, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO-NEXT: DW_CFA_expression: reg78 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -56, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO-NEXT: DW_CFA_expression: reg79 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -64, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# +# UNWINDINFO: DW_CFA_restore_extended: reg104 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg105 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg106 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg107 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg108 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg109 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg110 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg111 +# UNWINDINFO: DW_CFA_def_cfa: reg31 +16 +# UNWINDINFO: DW_CFA_def_cfa_offset: +0 +# UNWINDINFO-NEXT: DW_CFA_restore: reg30 +# UNWINDINFO-NEXT: DW_CFA_restore: reg29 + name: save_restore_sve_realign stack: - { id: 0, stack-id: scalable-vector, size: 16, alignment: 16 } @@ -845,6 +1082,19 @@ # CHECK-NEXT: frame-setup CFI_INSTRUCTION # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -7 # CHECK-NEXT: frame-setup CFI_INSTRUCTION + +# CHECK: $sp = frame-destroy ADDVL_XXI $sp, 7 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION +# CHECK-NEXT: $p15 = frame-destroy LDR_PXI $sp, 6 +# CHECK-NEXT: $p4 = frame-destroy LDR_PXI $sp, 7 +# CHECK-NEXT: $z23 = frame-destroy LDR_ZXI $sp, 1 +# CHECK-NEXT: $z8 = frame-destroy LDR_ZXI $sp, 2 +# CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 3 +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION +# CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.10) +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION +# CHECK-NEXT: frame-destroy CFI_INSTRUCTION # # ASM-LABEL: frame_layout: # ASM: .cfi_def_cfa_offset 16 @@ -853,11 +1103,24 @@ # ASM: .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG # ASM: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xd0, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 80 * VG # +# ASM: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG +# ASM: .cfi_def_cfa wsp, 16 +# ASM-NEXT: .cfi_restore z8 +# ASM: .cfi_def_cfa_offset 0 +# ASM-NEXT: .cfi_restore w29 + # UNWINDINFO: DW_CFA_def_cfa_offset: +16 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16 # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO: DW_CFA_expression: reg72 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus # UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +80, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# +# UNWINDINFO: DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus +# UNWINDINFO: DW_CFA_def_cfa: reg31 +16 +# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg104 +# UNWINDINFO: DW_CFA_def_cfa_offset: +0 +# UNWINDINFO-NEXT: DW_CFA_restore: reg29 + name: frame_layout stack: - { id: 0, type: default, size: 32, alignment: 16, stack-id: scalable-vector } Index: llvm/test/CodeGen/AArch64/framelayout-unaligned-fp.ll =================================================================== --- llvm/test/CodeGen/AArch64/framelayout-unaligned-fp.ll +++ llvm/test/CodeGen/AArch64/framelayout-unaligned-fp.ll @@ -11,7 +11,7 @@ @a = global i64 0, align 4 -define i64 @b() { +define i64 @b() uwtable { entry: %call = tail call i64 @d() %0 = alloca i8, i64 ptrtoint (i64 ()* @d to i64), align 16 @@ -33,9 +33,15 @@ ; CHECK-NEXT: add x29, sp, #8 ; CHECK: sub sp, x29, #8 +; CHECK-NEXT: .cfi_def_cfa wsp, 32 ; CHECK-NEXT: ldr x19, [sp, #24] ; CHECK-NEXT: ldp x29, x30, [sp, #8] ; CHECK-NEXT: ldr d8, [sp], #32 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w19 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 +; CHECK-NEXT: .cfi_restore b8 ; CHECK-NEXT: ret declare i64 @d() Index: llvm/test/CodeGen/AArch64/large-stack.ll =================================================================== --- llvm/test/CodeGen/AArch64/large-stack.ll +++ llvm/test/CodeGen/AArch64/large-stack.ll @@ -21,7 +21,7 @@ declare dso_local i32 @printf(i8*, ...) -attributes #0 = { noinline optnone "frame-pointer"="all" } +attributes #0 = { noinline optnone "frame-pointer"="all" uwtable } ; CHECK: stp x[[SPILL_REG1:[0-9]+]], x[[SPILL_REG2:[0-9]+]], [sp, #-[[SPILL_OFFSET1:[0-9]+]]] ; CHECK-NEXT: .cfi_def_cfa_offset [[SPILL_OFFSET1]] @@ -48,5 +48,10 @@ ; CHECK-COUNT-128: add sp, sp, #[[STACK1]], lsl #12 ; CHECK-NEXT: add sp, sp, #[[STACK2]], lsl #12 ; CHECK-NEXT: add sp, sp, #[[STACK3]] +; CHECK-NEXT: .cfi_def_cfa wsp, [[SPILL_OFFSET1]] ; CHECK-NEXT: ldr x[[SPILL_REG3]], [sp, #[[SPILL_OFFSET2]]] ; CHECK-NEXT: ldp x[[SPILL_REG1]], x[[SPILL_REG2]], [sp], #[[SPILL_OFFSET1]] +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w[[SPILL_REG3]] +; CHECK-NEXT: .cfi_restore w[[SPILL_REG2]] +; CHECK-NEXT: .cfi_restore w[[SPILL_REG1]] Index: llvm/test/CodeGen/AArch64/local_vars.ll =================================================================== --- llvm/test/CodeGen/AArch64/local_vars.ll +++ llvm/test/CodeGen/AArch64/local_vars.ll @@ -23,7 +23,7 @@ ret void } -define void @trivial_fp_func() { +define void @trivial_fp_func() uwtable { ; CHECK-LABEL: trivial_fp_func: ; CHECK: str x30, [sp, #-16]! ; CHECK-NOT: mov x29, sp @@ -43,9 +43,15 @@ ret void ; CHECK: ldr x30, [sp], #16 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret -; CHECK-WITHFP-ARM64: ldp x29, x30, [sp], #16 +; CHECK-WITHFP-ARM64: .cfi_def_cfa wsp, 16 +; CHECK-WITHFP-ARM64-NEXT: ldp x29, x30, [sp], #16 +; CHECK-WITHFP-ARM64-NEXT: .cfi_def_cfa_offset 0 +; CHECK-WITHFP-ARM64-NEXT: .cfi_restore w30 +; CHECK-WITHFP-ARM64-NEXT: .cfi_restore w29 ; CHECK-WITHFP-ARM64-NEXT: ret } Index: llvm/test/CodeGen/AArch64/machine-outliner-remarks.ll =================================================================== --- llvm/test/CodeGen/AArch64/machine-outliner-remarks.ll +++ llvm/test/CodeGen/AArch64/machine-outliner-remarks.ll @@ -102,7 +102,7 @@ ret void } -attributes #0 = { noredzone nounwind ssp uwtable "frame-pointer"="none" "target-cpu"="cyclone" } +attributes #0 = { noredzone nounwind ssp uwtable minsize "frame-pointer"="none" "target-cpu"="cyclone" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4, !5, !6} Index: llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll =================================================================== --- llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll +++ llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll @@ -70,7 +70,7 @@ ret i32 %add } -attributes #0 = { "sign-return-address"="non-leaf" } +attributes #0 = { "sign-return-address"="non-leaf" minsize } ; CHECK-NOT: OUTLINED_FUNCTION_{{.*}} ; CHECK-NOT: .cfi_b_key_frame Index: llvm/test/CodeGen/AArch64/merge-store-dependency.ll =================================================================== --- llvm/test/CodeGen/AArch64/merge-store-dependency.ll +++ llvm/test/CodeGen/AArch64/merge-store-dependency.ll @@ -6,7 +6,7 @@ @gv0 = internal unnamed_addr global i32 0, align 4 @gv1 = internal unnamed_addr global %struct1** null, align 8 -define void @test(%struct1* %fde, i32 %fd, void (i32, i32, i8*)* %func, i8* %arg) { +define void @test(%struct1* %fde, i32 %fd, void (i32, i32, i8*)* %func, i8* %arg) uwtable { ;CHECK-LABEL: test ; A53-LABEL: test: ; A53: // %bb.0: // %entry @@ -45,6 +45,9 @@ ; A53-NEXT: adrp x8, gv1 ; A53-NEXT: str x0, [x8, :lo12:gv1] ; A53-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload +; A53-NEXT: .cfi_def_cfa_offset 0 +; A53-NEXT: .cfi_restore w19 +; A53-NEXT: .cfi_restore w30 ; A53-NEXT: ret ; A53-NEXT: .LBB0_4: // %while.body.i.split ; A53-NEXT: // =>This Inner Loop Header: Depth=1 Index: llvm/test/CodeGen/AArch64/optimize-cond-branch.ll =================================================================== --- llvm/test/CodeGen/AArch64/optimize-cond-branch.ll +++ llvm/test/CodeGen/AArch64/optimize-cond-branch.ll @@ -10,7 +10,7 @@ ; formed in SelectionDAG, optimizeCondBranch() only triggers if the and ; instruction is in a different block than the conditional jump. -define void @func() { +define void @func() uwtable { ; CHECK-LABEL: func: ; CHECK: // %bb.0: ; CHECK-NEXT: mov w8, #1 @@ -24,12 +24,16 @@ ; CHECK-NEXT: ldr w8, [x8] ; CHECK-NEXT: and w0, w8, #0x100 ; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: cbz w0, .LBB0_5 ; CHECK-NEXT: .LBB0_3: // %common.ret.sink.split ; CHECK-NEXT: b extfunc ; CHECK-NEXT: .LBB0_4: // %b2 ; CHECK-NEXT: bl extfunc ; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: cbnz w0, .LBB0_3 ; CHECK-NEXT: .LBB0_5: // %common.ret ; CHECK-NEXT: ret Index: llvm/test/CodeGen/AArch64/ragreedy-local-interval-cost.ll =================================================================== --- llvm/test/CodeGen/AArch64/ragreedy-local-interval-cost.ll +++ llvm/test/CodeGen/AArch64/ragreedy-local-interval-cost.ll @@ -5,7 +5,7 @@ @B = external dso_local local_unnamed_addr global [8 x [8 x i64]], align 8 @C = external dso_local local_unnamed_addr global [8 x [8 x i64]], align 8 -define dso_local void @run_test() local_unnamed_addr #0 { +define dso_local void @run_test() local_unnamed_addr uwtable { ; CHECK-LABEL: run_test: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: sub sp, sp, #96 @@ -175,6 +175,15 @@ ; CHECK-NEXT: stp q4, q3, [x8, #432] ; CHECK-NEXT: str q0, [x8, #496] ; CHECK-NEXT: add sp, sp, #96 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore b8 +; CHECK-NEXT: .cfi_restore b9 +; CHECK-NEXT: .cfi_restore b10 +; CHECK-NEXT: .cfi_restore b11 +; CHECK-NEXT: .cfi_restore b12 +; CHECK-NEXT: .cfi_restore b13 +; CHECK-NEXT: .cfi_restore b14 +; CHECK-NEXT: .cfi_restore b15 ; CHECK-NEXT: ret ; CH`ECK-NEXT: .cfi_offset b9, -16 entry: Index: llvm/test/CodeGen/AArch64/settag.ll =================================================================== --- llvm/test/CodeGen/AArch64/settag.ll +++ llvm/test/CodeGen/AArch64/settag.ll @@ -116,12 +116,13 @@ ret void } -define void @stg_alloca1() { +define void @stg_alloca1() uwtable { ; CHECK-LABEL: stg_alloca1: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: sub sp, sp, #16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: stg sp, [sp], #16 +; CHECK-NEXT: .cfi_def_cfa_offset 0 ; CHECK-NEXT: ret entry: %a = alloca i8, i32 16, align 16 @@ -129,7 +130,7 @@ ret void } -define void @stg_alloca5() { +define void @stg_alloca5() uwtable { ; CHECK-LABEL: stg_alloca5: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: sub sp, sp, #80 @@ -137,6 +138,7 @@ ; CHECK-NEXT: st2g sp, [sp, #32] ; CHECK-NEXT: stg sp, [sp, #64] ; CHECK-NEXT: st2g sp, [sp], #80 +; CHECK-NEXT: .cfi_def_cfa_offset 0 ; CHECK-NEXT: ret entry: %a = alloca i8, i32 80, align 16 @@ -144,7 +146,7 @@ ret void } -define void @stg_alloca17() { +define void @stg_alloca17() uwtable { ; CHECK-LABEL: stg_alloca17: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: sub sp, sp, #288 @@ -159,7 +161,10 @@ ; CHECK-NEXT: cbnz x8, .LBB11_1 ; CHECK-NEXT: // %bb.2: // %entry ; CHECK-NEXT: stg sp, [sp], #16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret entry: %a = alloca i8, i32 272, align 16 Index: llvm/test/CodeGen/AArch64/shrink-wrapping-vla.ll =================================================================== --- llvm/test/CodeGen/AArch64/shrink-wrapping-vla.ll +++ llvm/test/CodeGen/AArch64/shrink-wrapping-vla.ll @@ -15,7 +15,7 @@ ; ; RUN: llc -mtriple aarch64-linux %s -o - | FileCheck %s -define dso_local void @f(i32 %n, i32* nocapture %x) { +define dso_local void @f(i32 %n, i32* nocapture %x) uwtable { entry: %cmp = icmp slt i32 %n, 0 br i1 %cmp, label %return, label %if.end @@ -80,6 +80,10 @@ ; CHECK: stp x29, x30, [sp, #-16]! ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: mov x29, sp +; CHECK-NEXT: .cfi_def_cfa w29, 16 +; CHECK-NEXT: .cfi_offset w30, -8 +; CHECK-NEXT: .cfi_offset w29, -16 + ; VLA allocation ; CHECK: mov [[X2:x[0-9]+]], sp @@ -93,4 +97,9 @@ ; CHECK: mov sp, [[SAVE]] ; Epilogue ; CHECK-NEXT: mov sp, x29 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldp x29, x30, [sp], #16 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 +; CHECK-NEXT: ret Index: llvm/test/CodeGen/AArch64/speculation-hardening-loads.ll =================================================================== --- llvm/test/CodeGen/AArch64/speculation-hardening-loads.ll +++ llvm/test/CodeGen/AArch64/speculation-hardening-loads.ll @@ -138,7 +138,7 @@ ret <2 x double> %vld1_lane } -define i32 @deadload() speculative_load_hardening { +define i32 @deadload() speculative_load_hardening uwtable { entry: ; CHECK-LABEL: deadload ; CHECK: cmp sp, #0 @@ -147,6 +147,7 @@ ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: ldr w8, [sp, #12] ; CHECK-NEXT: add sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 0 ; CHECK-NEXT: mov [[TMPREG:x[0-9]+]], sp ; CHECK-NEXT: and [[TMPREG]], [[TMPREG]], x16 ; CHECK-NEXT: mov sp, [[TMPREG]] Index: llvm/test/CodeGen/AArch64/split-vector-insert.ll =================================================================== --- llvm/test/CodeGen/AArch64/split-vector-insert.ll +++ llvm/test/CodeGen/AArch64/split-vector-insert.ll @@ -3,7 +3,7 @@ ; REQUIRES: asserts target triple = "aarch64-unknown-linux-gnu" -attributes #0 = {"target-features"="+sve"} +attributes #0 = {"target-features"="+sve" uwtable} declare @llvm.experimental.vector.insert.nxv2i64.v8i64(, <8 x i64>, i64) declare @llvm.experimental.vector.insert.nxv2f64.v8f64(, <8 x double>, i64) @@ -52,7 +52,10 @@ ; CHECK-NEXT: str q4, [x9, x8] ; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #3, mul vl] ; CHECK-NEXT: addvl sp, sp, #4 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret @@ -104,7 +107,10 @@ ; CHECK-NEXT: str q4, [x9, x8] ; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #3, mul vl] ; CHECK-NEXT: addvl sp, sp, #4 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret Index: llvm/test/CodeGen/AArch64/stack-guard-sysreg.ll =================================================================== --- llvm/test/CodeGen/AArch64/stack-guard-sysreg.ll +++ llvm/test/CodeGen/AArch64/stack-guard-sysreg.ll @@ -81,7 +81,11 @@ ; CHECK-NEXT: b.ne .LBB0_2 ; CHECK-NEXT: // %bb.1: // %entry ; CHECK-NEXT: mov sp, x29 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret ; CHECK-NEXT: .LBB0_2: // %entry ; CHECK-NEXT: bl __stack_chk_fail @@ -96,7 +100,7 @@ ; CHECK-BAD-OFFSET: LLVM ERROR: Unable to encode Stack Protector Guard Offset -attributes #0 = { sspstrong } +attributes #0 = { sspstrong uwtable } !llvm.module.flags = !{!1, !2, !3} !1 = !{i32 2, !"stack-protector-guard", !"sysreg"} Index: llvm/test/CodeGen/AArch64/storepairsuppress_minsize.ll =================================================================== --- llvm/test/CodeGen/AArch64/storepairsuppress_minsize.ll +++ llvm/test/CodeGen/AArch64/storepairsuppress_minsize.ll @@ -7,7 +7,7 @@ declare %T_IN_BLOCK @return_in_block() @in_block_store = dso_local global %T_IN_BLOCK zeroinitializer, align 8 -define void @test_default() { +define void @test_default() uwtable { ; CHECK-LABEL: test_default: ; CHECK: // %bb.0: ; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill @@ -23,13 +23,15 @@ ; CHECK-NEXT: str d4, [x8, #32] ; CHECK-NEXT: str d5, [x8, #40] ; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret %1 = call %T_IN_BLOCK @return_in_block() store %T_IN_BLOCK %1, %T_IN_BLOCK* @in_block_store ret void } -define void @test_minsize() minsize { +define void @test_minsize() minsize uwtable { ; CHECK-LABEL: test_minsize: ; CHECK: // %bb.0: ; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill @@ -48,7 +50,7 @@ ret void } -define void @test_optsize() optsize { +define void @test_optsize() optsize uwtable { ; CHECK-LABEL: test_optsize: ; CHECK: // %bb.0: ; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill @@ -61,6 +63,8 @@ ; CHECK-NEXT: stp d2, d3, [x8, #16] ; CHECK-NEXT: stp d4, d5, [x8, #32] ; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret %1 = call %T_IN_BLOCK @return_in_block() store %T_IN_BLOCK %1, %T_IN_BLOCK* @in_block_store Index: llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll =================================================================== --- llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll +++ llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll @@ -56,7 +56,7 @@ ret %res } -define @extract_nxv14i1_nxv28i1_14( %in) { +define @extract_nxv14i1_nxv28i1_14( %in) uwtable { ; CHECK-LABEL: extract_nxv14i1_nxv28i1_14: ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill @@ -90,7 +90,10 @@ ; CHECK-NEXT: uzp1 p0.h, p0.h, p3.h ; CHECK-NEXT: uzp1 p0.b, p0.b, p1.b ; CHECK-NEXT: addvl sp, sp, #1 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %res = call @llvm.experimental.vector.extract.nxv14i1.nxv28i1( %in, i64 14) ret %res Index: llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll =================================================================== --- llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll +++ llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -aarch64-sve-vector-bits-min=128 < %s | FileCheck %s -D#VBYTES=16 -check-prefix=NO_SVE ; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -D#VBYTES=32 ; RUN: llc -aarch64-sve-vector-bits-min=384 < %s | FileCheck %s -D#VBYTES=32 @@ -22,6 +23,14 @@ ; Don't use SVE for 64-bit vectors. define <4 x half> @select_v4f16(<4 x half> %op1, <4 x half> %op2, i1 %mask) #0 { +; NO_SVE-LABEL: select_v4f16: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w0, #0x1 +; NO_SVE-NEXT: csetm w8, ne +; NO_SVE-NEXT: dup v2.4h, w8 +; NO_SVE-NEXT: bif v0.8b, v1.8b, v2.8b +; NO_SVE-NEXT: ret +; ; CHECK-LABEL: select_v4f16: ; CHECK: // %bb.0: ; CHECK-NEXT: tst w0, #0x1 @@ -35,6 +44,14 @@ ; Don't use SVE for 128-bit vectors. define <8 x half> @select_v8f16(<8 x half> %op1, <8 x half> %op2, i1 %mask) #0 { +; NO_SVE-LABEL: select_v8f16: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w0, #0x1 +; NO_SVE-NEXT: csetm w8, ne +; NO_SVE-NEXT: dup v2.8h, w8 +; NO_SVE-NEXT: bif v0.16b, v1.16b, v2.16b +; NO_SVE-NEXT: ret +; ; CHECK-LABEL: select_v8f16: ; CHECK: // %bb.0: ; CHECK-NEXT: tst w0, #0x1 @@ -47,6 +64,20 @@ } define void @select_v16f16(<16 x half>* %a, <16 x half>* %b, i1 %mask) #0 { +; NO_SVE-LABEL: select_v16f16: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w2, #0x1 +; NO_SVE-NEXT: ldr q0, [x0] +; NO_SVE-NEXT: csetm w8, ne +; NO_SVE-NEXT: ldr q1, [x0, #16] +; NO_SVE-NEXT: ldr q2, [x1] +; NO_SVE-NEXT: ldr q3, [x1, #16] +; NO_SVE-NEXT: dup v4.8h, w8 +; NO_SVE-NEXT: bif v0.16b, v2.16b, v4.16b +; NO_SVE-NEXT: bif v1.16b, v3.16b, v4.16b +; NO_SVE-NEXT: stp q0, q1, [x0] +; NO_SVE-NEXT: ret +; ; CHECK-LABEL: select_v16f16: ; CHECK: // %bb.0: ; CHECK-NEXT: and w8, w2, #0x1 @@ -68,6 +99,27 @@ } define void @select_v32f16(<32 x half>* %a, <32 x half>* %b, i1 %mask) #0 { +; NO_SVE-LABEL: select_v32f16: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w2, #0x1 +; NO_SVE-NEXT: ldr q0, [x0, #48] +; NO_SVE-NEXT: csetm w8, ne +; NO_SVE-NEXT: ldr q1, [x0] +; NO_SVE-NEXT: ldr q2, [x0, #16] +; NO_SVE-NEXT: ldr q3, [x0, #32] +; NO_SVE-NEXT: ldr q4, [x1, #48] +; NO_SVE-NEXT: dup v6.8h, w8 +; NO_SVE-NEXT: ldr q5, [x1] +; NO_SVE-NEXT: ldr q7, [x1, #16] +; NO_SVE-NEXT: ldr q16, [x1, #32] +; NO_SVE-NEXT: bif v1.16b, v5.16b, v6.16b +; NO_SVE-NEXT: bif v2.16b, v7.16b, v6.16b +; NO_SVE-NEXT: bif v0.16b, v4.16b, v6.16b +; NO_SVE-NEXT: bif v3.16b, v16.16b, v6.16b +; NO_SVE-NEXT: stp q1, q2, [x0] +; NO_SVE-NEXT: stp q3, q0, [x0, #32] +; NO_SVE-NEXT: ret +; ; VBITS_GE_512-LABEL: select_v32f16: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: and w8, w2, #0x1 @@ -89,6 +141,44 @@ } define void @select_v64f16(<64 x half>* %a, <64 x half>* %b, i1 %mask) #0 { +; NO_SVE-LABEL: select_v64f16: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w2, #0x1 +; NO_SVE-NEXT: ldr q0, [x0, #16] +; NO_SVE-NEXT: csetm w8, ne +; NO_SVE-NEXT: ldr q1, [x0] +; NO_SVE-NEXT: ldr q2, [x0, #48] +; NO_SVE-NEXT: ldr q3, [x0, #32] +; NO_SVE-NEXT: ldr q4, [x0, #80] +; NO_SVE-NEXT: dup v21.8h, w8 +; NO_SVE-NEXT: ldr q5, [x0, #64] +; NO_SVE-NEXT: ldr q6, [x0, #112] +; NO_SVE-NEXT: ldr q7, [x0, #96] +; NO_SVE-NEXT: ldr q16, [x1, #16] +; NO_SVE-NEXT: ldr q17, [x1] +; NO_SVE-NEXT: ldr q18, [x1, #48] +; NO_SVE-NEXT: ldr q19, [x1, #32] +; NO_SVE-NEXT: bif v0.16b, v16.16b, v21.16b +; NO_SVE-NEXT: ldr q20, [x1, #80] +; NO_SVE-NEXT: bif v1.16b, v17.16b, v21.16b +; NO_SVE-NEXT: ldr q16, [x1, #64] +; NO_SVE-NEXT: bif v2.16b, v18.16b, v21.16b +; NO_SVE-NEXT: ldr q17, [x1, #112] +; NO_SVE-NEXT: bif v3.16b, v19.16b, v21.16b +; NO_SVE-NEXT: ldr q18, [x1, #96] +; NO_SVE-NEXT: bif v4.16b, v20.16b, v21.16b +; NO_SVE-NEXT: stp q1, q0, [x0] +; NO_SVE-NEXT: mov v0.16b, v21.16b +; NO_SVE-NEXT: mov v1.16b, v21.16b +; NO_SVE-NEXT: stp q3, q2, [x0, #32] +; NO_SVE-NEXT: mov v2.16b, v21.16b +; NO_SVE-NEXT: bsl v0.16b, v5.16b, v16.16b +; NO_SVE-NEXT: bsl v1.16b, v6.16b, v17.16b +; NO_SVE-NEXT: bsl v2.16b, v7.16b, v18.16b +; NO_SVE-NEXT: stp q0, q4, [x0, #64] +; NO_SVE-NEXT: stp q2, q1, [x0, #96] +; NO_SVE-NEXT: ret +; ; VBITS_GE_1024-LABEL: select_v64f16: ; VBITS_GE_1024: // %bb.0: ; VBITS_GE_1024-NEXT: and w8, w2, #0x1 @@ -110,6 +200,89 @@ } define void @select_v128f16(<128 x half>* %a, <128 x half>* %b, i1 %mask) #0 { +; NO_SVE-LABEL: select_v128f16: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: stp d11, d10, [sp, #-32]! // 16-byte Folded Spill +; NO_SVE-NEXT: .cfi_def_cfa_offset 32 +; NO_SVE-NEXT: stp d9, d8, [sp, #16] // 16-byte Folded Spill +; NO_SVE-NEXT: .cfi_offset b8, -8 +; NO_SVE-NEXT: .cfi_offset b9, -16 +; NO_SVE-NEXT: .cfi_offset b10, -24 +; NO_SVE-NEXT: .cfi_offset b11, -32 +; NO_SVE-NEXT: tst w2, #0x1 +; NO_SVE-NEXT: ldr q0, [x0, #240] +; NO_SVE-NEXT: csetm w8, ne +; NO_SVE-NEXT: ldr q1, [x0, #224] +; NO_SVE-NEXT: ldr q2, [x0, #208] +; NO_SVE-NEXT: ldr q3, [x0, #192] +; NO_SVE-NEXT: ldr q4, [x0, #176] +; NO_SVE-NEXT: dup v8.8h, w8 +; NO_SVE-NEXT: ldr q5, [x0, #160] +; NO_SVE-NEXT: ldr q6, [x0, #144] +; NO_SVE-NEXT: ldr q7, [x0, #128] +; NO_SVE-NEXT: ldr q16, [x0, #112] +; NO_SVE-NEXT: ldr q17, [x0, #96] +; NO_SVE-NEXT: ldr q18, [x0, #80] +; NO_SVE-NEXT: ldr q19, [x0, #64] +; NO_SVE-NEXT: ldr q20, [x0, #48] +; NO_SVE-NEXT: ldr q21, [x0, #32] +; NO_SVE-NEXT: ldr q22, [x0, #16] +; NO_SVE-NEXT: ldr q23, [x0] +; NO_SVE-NEXT: ldr q24, [x1, #240] +; NO_SVE-NEXT: ldr q25, [x1, #224] +; NO_SVE-NEXT: ldr q26, [x1, #208] +; NO_SVE-NEXT: ldr q27, [x1, #192] +; NO_SVE-NEXT: bif v0.16b, v24.16b, v8.16b +; NO_SVE-NEXT: ldr q28, [x1, #176] +; NO_SVE-NEXT: bif v1.16b, v25.16b, v8.16b +; NO_SVE-NEXT: ldr q29, [x1, #160] +; NO_SVE-NEXT: bif v2.16b, v26.16b, v8.16b +; NO_SVE-NEXT: ldr q30, [x1, #144] +; NO_SVE-NEXT: bif v3.16b, v27.16b, v8.16b +; NO_SVE-NEXT: ldr q31, [x1, #128] +; NO_SVE-NEXT: ldr q9, [x1, #112] +; NO_SVE-NEXT: ldr q10, [x1, #96] +; NO_SVE-NEXT: bif v4.16b, v28.16b, v8.16b +; NO_SVE-NEXT: ldr q28, [x1, #80] +; NO_SVE-NEXT: ldr q24, [x1, #64] +; NO_SVE-NEXT: ldr q25, [x1, #48] +; NO_SVE-NEXT: ldr q26, [x1, #32] +; NO_SVE-NEXT: ldr q27, [x1, #16] +; NO_SVE-NEXT: ldr q11, [x1] +; NO_SVE-NEXT: stp q1, q0, [x0, #224] +; NO_SVE-NEXT: mov v0.16b, v8.16b +; NO_SVE-NEXT: stp q3, q2, [x0, #192] +; NO_SVE-NEXT: mov v1.16b, v8.16b +; NO_SVE-NEXT: mov v2.16b, v8.16b +; NO_SVE-NEXT: mov v3.16b, v8.16b +; NO_SVE-NEXT: bsl v0.16b, v5.16b, v29.16b +; NO_SVE-NEXT: bsl v1.16b, v6.16b, v30.16b +; NO_SVE-NEXT: bsl v2.16b, v7.16b, v31.16b +; NO_SVE-NEXT: bsl v3.16b, v16.16b, v9.16b +; NO_SVE-NEXT: stp q0, q4, [x0, #160] +; NO_SVE-NEXT: mov v4.16b, v8.16b +; NO_SVE-NEXT: mov v0.16b, v8.16b +; NO_SVE-NEXT: stp q2, q1, [x0, #128] +; NO_SVE-NEXT: mov v1.16b, v8.16b +; NO_SVE-NEXT: mov v2.16b, v8.16b +; NO_SVE-NEXT: bsl v4.16b, v17.16b, v10.16b +; NO_SVE-NEXT: bsl v0.16b, v18.16b, v28.16b +; NO_SVE-NEXT: bsl v1.16b, v19.16b, v24.16b +; NO_SVE-NEXT: bsl v2.16b, v20.16b, v25.16b +; NO_SVE-NEXT: stp q4, q3, [x0, #96] +; NO_SVE-NEXT: mov v3.16b, v8.16b +; NO_SVE-NEXT: mov v4.16b, v8.16b +; NO_SVE-NEXT: stp q1, q0, [x0, #64] +; NO_SVE-NEXT: mov v0.16b, v8.16b +; NO_SVE-NEXT: ldp d9, d8, [sp, #16] // 16-byte Folded Reload +; NO_SVE-NEXT: bsl v3.16b, v21.16b, v26.16b +; NO_SVE-NEXT: bsl v4.16b, v22.16b, v27.16b +; NO_SVE-NEXT: bsl v0.16b, v23.16b, v11.16b +; NO_SVE-NEXT: stp q3, q2, [x0, #32] +; NO_SVE-NEXT: stp q0, q4, [x0] +; NO_SVE-NEXT: ldp d11, d10, [sp], #32 // 16-byte Folded Reload +; NO_SVE-NEXT: ret +; ; VBITS_GE_2048-LABEL: select_v128f16: ; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048-NEXT: and w8, w2, #0x1 @@ -132,6 +305,14 @@ ; Don't use SVE for 64-bit vectors. define <2 x float> @select_v2f32(<2 x float> %op1, <2 x float> %op2, i1 %mask) #0 { +; NO_SVE-LABEL: select_v2f32: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w0, #0x1 +; NO_SVE-NEXT: csetm w8, ne +; NO_SVE-NEXT: dup v2.2s, w8 +; NO_SVE-NEXT: bif v0.8b, v1.8b, v2.8b +; NO_SVE-NEXT: ret +; ; CHECK-LABEL: select_v2f32: ; CHECK: // %bb.0: ; CHECK-NEXT: tst w0, #0x1 @@ -145,6 +326,14 @@ ; Don't use SVE for 128-bit vectors. define <4 x float> @select_v4f32(<4 x float> %op1, <4 x float> %op2, i1 %mask) #0 { +; NO_SVE-LABEL: select_v4f32: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w0, #0x1 +; NO_SVE-NEXT: csetm w8, ne +; NO_SVE-NEXT: dup v2.4s, w8 +; NO_SVE-NEXT: bif v0.16b, v1.16b, v2.16b +; NO_SVE-NEXT: ret +; ; CHECK-LABEL: select_v4f32: ; CHECK: // %bb.0: ; CHECK-NEXT: tst w0, #0x1 @@ -157,6 +346,20 @@ } define void @select_v8f32(<8 x float>* %a, <8 x float>* %b, i1 %mask) #0 { +; NO_SVE-LABEL: select_v8f32: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w2, #0x1 +; NO_SVE-NEXT: ldr q0, [x0] +; NO_SVE-NEXT: csetm w8, ne +; NO_SVE-NEXT: ldr q1, [x0, #16] +; NO_SVE-NEXT: ldr q2, [x1] +; NO_SVE-NEXT: ldr q3, [x1, #16] +; NO_SVE-NEXT: dup v4.4s, w8 +; NO_SVE-NEXT: bif v0.16b, v2.16b, v4.16b +; NO_SVE-NEXT: bif v1.16b, v3.16b, v4.16b +; NO_SVE-NEXT: stp q0, q1, [x0] +; NO_SVE-NEXT: ret +; ; CHECK-LABEL: select_v8f32: ; CHECK: // %bb.0: ; CHECK-NEXT: and w8, w2, #0x1 @@ -178,6 +381,27 @@ } define void @select_v16f32(<16 x float>* %a, <16 x float>* %b, i1 %mask) #0 { +; NO_SVE-LABEL: select_v16f32: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w2, #0x1 +; NO_SVE-NEXT: ldr q0, [x0, #48] +; NO_SVE-NEXT: csetm w8, ne +; NO_SVE-NEXT: ldr q1, [x0] +; NO_SVE-NEXT: ldr q2, [x0, #16] +; NO_SVE-NEXT: ldr q3, [x0, #32] +; NO_SVE-NEXT: ldr q4, [x1, #48] +; NO_SVE-NEXT: dup v6.4s, w8 +; NO_SVE-NEXT: ldr q5, [x1] +; NO_SVE-NEXT: ldr q7, [x1, #16] +; NO_SVE-NEXT: ldr q16, [x1, #32] +; NO_SVE-NEXT: bif v1.16b, v5.16b, v6.16b +; NO_SVE-NEXT: bif v2.16b, v7.16b, v6.16b +; NO_SVE-NEXT: bif v0.16b, v4.16b, v6.16b +; NO_SVE-NEXT: bif v3.16b, v16.16b, v6.16b +; NO_SVE-NEXT: stp q1, q2, [x0] +; NO_SVE-NEXT: stp q3, q0, [x0, #32] +; NO_SVE-NEXT: ret +; ; VBITS_GE_512-LABEL: select_v16f32: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: and w8, w2, #0x1 @@ -199,6 +423,44 @@ } define void @select_v32f32(<32 x float>* %a, <32 x float>* %b, i1 %mask) #0 { +; NO_SVE-LABEL: select_v32f32: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w2, #0x1 +; NO_SVE-NEXT: ldr q0, [x0, #16] +; NO_SVE-NEXT: csetm w8, ne +; NO_SVE-NEXT: ldr q1, [x0] +; NO_SVE-NEXT: ldr q2, [x0, #48] +; NO_SVE-NEXT: ldr q3, [x0, #32] +; NO_SVE-NEXT: ldr q4, [x0, #80] +; NO_SVE-NEXT: dup v21.4s, w8 +; NO_SVE-NEXT: ldr q5, [x0, #64] +; NO_SVE-NEXT: ldr q6, [x0, #112] +; NO_SVE-NEXT: ldr q7, [x0, #96] +; NO_SVE-NEXT: ldr q16, [x1, #16] +; NO_SVE-NEXT: ldr q17, [x1] +; NO_SVE-NEXT: ldr q18, [x1, #48] +; NO_SVE-NEXT: ldr q19, [x1, #32] +; NO_SVE-NEXT: bif v0.16b, v16.16b, v21.16b +; NO_SVE-NEXT: ldr q20, [x1, #80] +; NO_SVE-NEXT: bif v1.16b, v17.16b, v21.16b +; NO_SVE-NEXT: ldr q16, [x1, #64] +; NO_SVE-NEXT: bif v2.16b, v18.16b, v21.16b +; NO_SVE-NEXT: ldr q17, [x1, #112] +; NO_SVE-NEXT: bif v3.16b, v19.16b, v21.16b +; NO_SVE-NEXT: ldr q18, [x1, #96] +; NO_SVE-NEXT: bif v4.16b, v20.16b, v21.16b +; NO_SVE-NEXT: stp q1, q0, [x0] +; NO_SVE-NEXT: mov v0.16b, v21.16b +; NO_SVE-NEXT: mov v1.16b, v21.16b +; NO_SVE-NEXT: stp q3, q2, [x0, #32] +; NO_SVE-NEXT: mov v2.16b, v21.16b +; NO_SVE-NEXT: bsl v0.16b, v5.16b, v16.16b +; NO_SVE-NEXT: bsl v1.16b, v6.16b, v17.16b +; NO_SVE-NEXT: bsl v2.16b, v7.16b, v18.16b +; NO_SVE-NEXT: stp q0, q4, [x0, #64] +; NO_SVE-NEXT: stp q2, q1, [x0, #96] +; NO_SVE-NEXT: ret +; ; VBITS_GE_1024-LABEL: select_v32f32: ; VBITS_GE_1024: // %bb.0: ; VBITS_GE_1024-NEXT: and w8, w2, #0x1 @@ -220,6 +482,89 @@ } define void @select_v64f32(<64 x float>* %a, <64 x float>* %b, i1 %mask) #0 { +; NO_SVE-LABEL: select_v64f32: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: stp d11, d10, [sp, #-32]! // 16-byte Folded Spill +; NO_SVE-NEXT: .cfi_def_cfa_offset 32 +; NO_SVE-NEXT: stp d9, d8, [sp, #16] // 16-byte Folded Spill +; NO_SVE-NEXT: .cfi_offset b8, -8 +; NO_SVE-NEXT: .cfi_offset b9, -16 +; NO_SVE-NEXT: .cfi_offset b10, -24 +; NO_SVE-NEXT: .cfi_offset b11, -32 +; NO_SVE-NEXT: tst w2, #0x1 +; NO_SVE-NEXT: ldr q0, [x0, #240] +; NO_SVE-NEXT: csetm w8, ne +; NO_SVE-NEXT: ldr q1, [x0, #224] +; NO_SVE-NEXT: ldr q2, [x0, #208] +; NO_SVE-NEXT: ldr q3, [x0, #192] +; NO_SVE-NEXT: ldr q4, [x0, #176] +; NO_SVE-NEXT: dup v8.4s, w8 +; NO_SVE-NEXT: ldr q5, [x0, #160] +; NO_SVE-NEXT: ldr q6, [x0, #144] +; NO_SVE-NEXT: ldr q7, [x0, #128] +; NO_SVE-NEXT: ldr q16, [x0, #112] +; NO_SVE-NEXT: ldr q17, [x0, #96] +; NO_SVE-NEXT: ldr q18, [x0, #80] +; NO_SVE-NEXT: ldr q19, [x0, #64] +; NO_SVE-NEXT: ldr q20, [x0, #48] +; NO_SVE-NEXT: ldr q21, [x0, #32] +; NO_SVE-NEXT: ldr q22, [x0, #16] +; NO_SVE-NEXT: ldr q23, [x0] +; NO_SVE-NEXT: ldr q24, [x1, #240] +; NO_SVE-NEXT: ldr q25, [x1, #224] +; NO_SVE-NEXT: ldr q26, [x1, #208] +; NO_SVE-NEXT: ldr q27, [x1, #192] +; NO_SVE-NEXT: bif v0.16b, v24.16b, v8.16b +; NO_SVE-NEXT: ldr q28, [x1, #176] +; NO_SVE-NEXT: bif v1.16b, v25.16b, v8.16b +; NO_SVE-NEXT: ldr q29, [x1, #160] +; NO_SVE-NEXT: bif v2.16b, v26.16b, v8.16b +; NO_SVE-NEXT: ldr q30, [x1, #144] +; NO_SVE-NEXT: bif v3.16b, v27.16b, v8.16b +; NO_SVE-NEXT: ldr q31, [x1, #128] +; NO_SVE-NEXT: ldr q9, [x1, #112] +; NO_SVE-NEXT: ldr q10, [x1, #96] +; NO_SVE-NEXT: bif v4.16b, v28.16b, v8.16b +; NO_SVE-NEXT: ldr q28, [x1, #80] +; NO_SVE-NEXT: ldr q24, [x1, #64] +; NO_SVE-NEXT: ldr q25, [x1, #48] +; NO_SVE-NEXT: ldr q26, [x1, #32] +; NO_SVE-NEXT: ldr q27, [x1, #16] +; NO_SVE-NEXT: ldr q11, [x1] +; NO_SVE-NEXT: stp q1, q0, [x0, #224] +; NO_SVE-NEXT: mov v0.16b, v8.16b +; NO_SVE-NEXT: stp q3, q2, [x0, #192] +; NO_SVE-NEXT: mov v1.16b, v8.16b +; NO_SVE-NEXT: mov v2.16b, v8.16b +; NO_SVE-NEXT: mov v3.16b, v8.16b +; NO_SVE-NEXT: bsl v0.16b, v5.16b, v29.16b +; NO_SVE-NEXT: bsl v1.16b, v6.16b, v30.16b +; NO_SVE-NEXT: bsl v2.16b, v7.16b, v31.16b +; NO_SVE-NEXT: bsl v3.16b, v16.16b, v9.16b +; NO_SVE-NEXT: stp q0, q4, [x0, #160] +; NO_SVE-NEXT: mov v4.16b, v8.16b +; NO_SVE-NEXT: mov v0.16b, v8.16b +; NO_SVE-NEXT: stp q2, q1, [x0, #128] +; NO_SVE-NEXT: mov v1.16b, v8.16b +; NO_SVE-NEXT: mov v2.16b, v8.16b +; NO_SVE-NEXT: bsl v4.16b, v17.16b, v10.16b +; NO_SVE-NEXT: bsl v0.16b, v18.16b, v28.16b +; NO_SVE-NEXT: bsl v1.16b, v19.16b, v24.16b +; NO_SVE-NEXT: bsl v2.16b, v20.16b, v25.16b +; NO_SVE-NEXT: stp q4, q3, [x0, #96] +; NO_SVE-NEXT: mov v3.16b, v8.16b +; NO_SVE-NEXT: mov v4.16b, v8.16b +; NO_SVE-NEXT: stp q1, q0, [x0, #64] +; NO_SVE-NEXT: mov v0.16b, v8.16b +; NO_SVE-NEXT: ldp d9, d8, [sp, #16] // 16-byte Folded Reload +; NO_SVE-NEXT: bsl v3.16b, v21.16b, v26.16b +; NO_SVE-NEXT: bsl v4.16b, v22.16b, v27.16b +; NO_SVE-NEXT: bsl v0.16b, v23.16b, v11.16b +; NO_SVE-NEXT: stp q3, q2, [x0, #32] +; NO_SVE-NEXT: stp q0, q4, [x0] +; NO_SVE-NEXT: ldp d11, d10, [sp], #32 // 16-byte Folded Reload +; NO_SVE-NEXT: ret +; ; VBITS_GE_2048-LABEL: select_v64f32: ; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048-NEXT: and w8, w2, #0x1 @@ -242,6 +587,14 @@ ; Don't use SVE for 64-bit vectors. define <1 x double> @select_v1f64(<1 x double> %op1, <1 x double> %op2, i1 %mask) #0 { +; NO_SVE-LABEL: select_v1f64: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w0, #0x1 +; NO_SVE-NEXT: csetm x8, ne +; NO_SVE-NEXT: fmov d2, x8 +; NO_SVE-NEXT: bif v0.8b, v1.8b, v2.8b +; NO_SVE-NEXT: ret +; ; CHECK-LABEL: select_v1f64: ; CHECK: // %bb.0: ; CHECK-NEXT: tst w0, #0x1 @@ -255,6 +608,14 @@ ; Don't use SVE for 128-bit vectors. define <2 x double> @select_v2f64(<2 x double> %op1, <2 x double> %op2, i1 %mask) #0 { +; NO_SVE-LABEL: select_v2f64: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w0, #0x1 +; NO_SVE-NEXT: csetm x8, ne +; NO_SVE-NEXT: dup v2.2d, x8 +; NO_SVE-NEXT: bif v0.16b, v1.16b, v2.16b +; NO_SVE-NEXT: ret +; ; CHECK-LABEL: select_v2f64: ; CHECK: // %bb.0: ; CHECK-NEXT: tst w0, #0x1 @@ -267,6 +628,20 @@ } define void @select_v4f64(<4 x double>* %a, <4 x double>* %b, i1 %mask) #0 { +; NO_SVE-LABEL: select_v4f64: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w2, #0x1 +; NO_SVE-NEXT: ldr q0, [x0] +; NO_SVE-NEXT: csetm x8, ne +; NO_SVE-NEXT: ldr q1, [x0, #16] +; NO_SVE-NEXT: ldr q2, [x1] +; NO_SVE-NEXT: ldr q3, [x1, #16] +; NO_SVE-NEXT: dup v4.2d, x8 +; NO_SVE-NEXT: bif v0.16b, v2.16b, v4.16b +; NO_SVE-NEXT: bif v1.16b, v3.16b, v4.16b +; NO_SVE-NEXT: stp q0, q1, [x0] +; NO_SVE-NEXT: ret +; ; CHECK-LABEL: select_v4f64: ; CHECK: // %bb.0: ; CHECK-NEXT: and w8, w2, #0x1 @@ -288,6 +663,27 @@ } define void @select_v8f64(<8 x double>* %a, <8 x double>* %b, i1 %mask) #0 { +; NO_SVE-LABEL: select_v8f64: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w2, #0x1 +; NO_SVE-NEXT: ldr q0, [x0, #48] +; NO_SVE-NEXT: csetm x8, ne +; NO_SVE-NEXT: ldr q1, [x0] +; NO_SVE-NEXT: ldr q2, [x0, #16] +; NO_SVE-NEXT: ldr q3, [x0, #32] +; NO_SVE-NEXT: ldr q4, [x1, #48] +; NO_SVE-NEXT: dup v6.2d, x8 +; NO_SVE-NEXT: ldr q5, [x1] +; NO_SVE-NEXT: ldr q7, [x1, #16] +; NO_SVE-NEXT: ldr q16, [x1, #32] +; NO_SVE-NEXT: bif v1.16b, v5.16b, v6.16b +; NO_SVE-NEXT: bif v2.16b, v7.16b, v6.16b +; NO_SVE-NEXT: bif v0.16b, v4.16b, v6.16b +; NO_SVE-NEXT: bif v3.16b, v16.16b, v6.16b +; NO_SVE-NEXT: stp q1, q2, [x0] +; NO_SVE-NEXT: stp q3, q0, [x0, #32] +; NO_SVE-NEXT: ret +; ; VBITS_GE_512-LABEL: select_v8f64: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: and w8, w2, #0x1 @@ -309,6 +705,44 @@ } define void @select_v16f64(<16 x double>* %a, <16 x double>* %b, i1 %mask) #0 { +; NO_SVE-LABEL: select_v16f64: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: tst w2, #0x1 +; NO_SVE-NEXT: ldr q0, [x0, #16] +; NO_SVE-NEXT: csetm x8, ne +; NO_SVE-NEXT: ldr q1, [x0] +; NO_SVE-NEXT: ldr q2, [x0, #48] +; NO_SVE-NEXT: ldr q3, [x0, #32] +; NO_SVE-NEXT: ldr q4, [x0, #80] +; NO_SVE-NEXT: dup v21.2d, x8 +; NO_SVE-NEXT: ldr q5, [x0, #64] +; NO_SVE-NEXT: ldr q6, [x0, #112] +; NO_SVE-NEXT: ldr q7, [x0, #96] +; NO_SVE-NEXT: ldr q16, [x1, #16] +; NO_SVE-NEXT: ldr q17, [x1] +; NO_SVE-NEXT: ldr q18, [x1, #48] +; NO_SVE-NEXT: ldr q19, [x1, #32] +; NO_SVE-NEXT: bif v0.16b, v16.16b, v21.16b +; NO_SVE-NEXT: ldr q20, [x1, #80] +; NO_SVE-NEXT: bif v1.16b, v17.16b, v21.16b +; NO_SVE-NEXT: ldr q16, [x1, #64] +; NO_SVE-NEXT: bif v2.16b, v18.16b, v21.16b +; NO_SVE-NEXT: ldr q17, [x1, #112] +; NO_SVE-NEXT: bif v3.16b, v19.16b, v21.16b +; NO_SVE-NEXT: ldr q18, [x1, #96] +; NO_SVE-NEXT: bif v4.16b, v20.16b, v21.16b +; NO_SVE-NEXT: stp q1, q0, [x0] +; NO_SVE-NEXT: mov v0.16b, v21.16b +; NO_SVE-NEXT: mov v1.16b, v21.16b +; NO_SVE-NEXT: stp q3, q2, [x0, #32] +; NO_SVE-NEXT: mov v2.16b, v21.16b +; NO_SVE-NEXT: bsl v0.16b, v5.16b, v16.16b +; NO_SVE-NEXT: bsl v1.16b, v6.16b, v17.16b +; NO_SVE-NEXT: bsl v2.16b, v7.16b, v18.16b +; NO_SVE-NEXT: stp q0, q4, [x0, #64] +; NO_SVE-NEXT: stp q2, q1, [x0, #96] +; NO_SVE-NEXT: ret +; ; VBITS_GE_1024-LABEL: select_v16f64: ; VBITS_GE_1024: // %bb.0: ; VBITS_GE_1024-NEXT: and w8, w2, #0x1 @@ -330,6 +764,89 @@ } define void @select_v32f64(<32 x double>* %a, <32 x double>* %b, i1 %mask) #0 { +; NO_SVE-LABEL: select_v32f64: +; NO_SVE: // %bb.0: +; NO_SVE-NEXT: stp d11, d10, [sp, #-32]! // 16-byte Folded Spill +; NO_SVE-NEXT: .cfi_def_cfa_offset 32 +; NO_SVE-NEXT: stp d9, d8, [sp, #16] // 16-byte Folded Spill +; NO_SVE-NEXT: .cfi_offset b8, -8 +; NO_SVE-NEXT: .cfi_offset b9, -16 +; NO_SVE-NEXT: .cfi_offset b10, -24 +; NO_SVE-NEXT: .cfi_offset b11, -32 +; NO_SVE-NEXT: tst w2, #0x1 +; NO_SVE-NEXT: ldr q0, [x0, #240] +; NO_SVE-NEXT: csetm x8, ne +; NO_SVE-NEXT: ldr q1, [x0, #224] +; NO_SVE-NEXT: ldr q2, [x0, #208] +; NO_SVE-NEXT: ldr q3, [x0, #192] +; NO_SVE-NEXT: ldr q4, [x0, #176] +; NO_SVE-NEXT: dup v8.2d, x8 +; NO_SVE-NEXT: ldr q5, [x0, #160] +; NO_SVE-NEXT: ldr q6, [x0, #144] +; NO_SVE-NEXT: ldr q7, [x0, #128] +; NO_SVE-NEXT: ldr q16, [x0, #112] +; NO_SVE-NEXT: ldr q17, [x0, #96] +; NO_SVE-NEXT: ldr q18, [x0, #80] +; NO_SVE-NEXT: ldr q19, [x0, #64] +; NO_SVE-NEXT: ldr q20, [x0, #48] +; NO_SVE-NEXT: ldr q21, [x0, #32] +; NO_SVE-NEXT: ldr q22, [x0, #16] +; NO_SVE-NEXT: ldr q23, [x0] +; NO_SVE-NEXT: ldr q24, [x1, #240] +; NO_SVE-NEXT: ldr q25, [x1, #224] +; NO_SVE-NEXT: ldr q26, [x1, #208] +; NO_SVE-NEXT: ldr q27, [x1, #192] +; NO_SVE-NEXT: bif v0.16b, v24.16b, v8.16b +; NO_SVE-NEXT: ldr q28, [x1, #176] +; NO_SVE-NEXT: bif v1.16b, v25.16b, v8.16b +; NO_SVE-NEXT: ldr q29, [x1, #160] +; NO_SVE-NEXT: bif v2.16b, v26.16b, v8.16b +; NO_SVE-NEXT: ldr q30, [x1, #144] +; NO_SVE-NEXT: bif v3.16b, v27.16b, v8.16b +; NO_SVE-NEXT: ldr q31, [x1, #128] +; NO_SVE-NEXT: ldr q9, [x1, #112] +; NO_SVE-NEXT: ldr q10, [x1, #96] +; NO_SVE-NEXT: bif v4.16b, v28.16b, v8.16b +; NO_SVE-NEXT: ldr q28, [x1, #80] +; NO_SVE-NEXT: ldr q24, [x1, #64] +; NO_SVE-NEXT: ldr q25, [x1, #48] +; NO_SVE-NEXT: ldr q26, [x1, #32] +; NO_SVE-NEXT: ldr q27, [x1, #16] +; NO_SVE-NEXT: ldr q11, [x1] +; NO_SVE-NEXT: stp q1, q0, [x0, #224] +; NO_SVE-NEXT: mov v0.16b, v8.16b +; NO_SVE-NEXT: stp q3, q2, [x0, #192] +; NO_SVE-NEXT: mov v1.16b, v8.16b +; NO_SVE-NEXT: mov v2.16b, v8.16b +; NO_SVE-NEXT: mov v3.16b, v8.16b +; NO_SVE-NEXT: bsl v0.16b, v5.16b, v29.16b +; NO_SVE-NEXT: bsl v1.16b, v6.16b, v30.16b +; NO_SVE-NEXT: bsl v2.16b, v7.16b, v31.16b +; NO_SVE-NEXT: bsl v3.16b, v16.16b, v9.16b +; NO_SVE-NEXT: stp q0, q4, [x0, #160] +; NO_SVE-NEXT: mov v4.16b, v8.16b +; NO_SVE-NEXT: mov v0.16b, v8.16b +; NO_SVE-NEXT: stp q2, q1, [x0, #128] +; NO_SVE-NEXT: mov v1.16b, v8.16b +; NO_SVE-NEXT: mov v2.16b, v8.16b +; NO_SVE-NEXT: bsl v4.16b, v17.16b, v10.16b +; NO_SVE-NEXT: bsl v0.16b, v18.16b, v28.16b +; NO_SVE-NEXT: bsl v1.16b, v19.16b, v24.16b +; NO_SVE-NEXT: bsl v2.16b, v20.16b, v25.16b +; NO_SVE-NEXT: stp q4, q3, [x0, #96] +; NO_SVE-NEXT: mov v3.16b, v8.16b +; NO_SVE-NEXT: mov v4.16b, v8.16b +; NO_SVE-NEXT: stp q1, q0, [x0, #64] +; NO_SVE-NEXT: mov v0.16b, v8.16b +; NO_SVE-NEXT: ldp d9, d8, [sp, #16] // 16-byte Folded Reload +; NO_SVE-NEXT: bsl v3.16b, v21.16b, v26.16b +; NO_SVE-NEXT: bsl v4.16b, v22.16b, v27.16b +; NO_SVE-NEXT: bsl v0.16b, v23.16b, v11.16b +; NO_SVE-NEXT: stp q3, q2, [x0, #32] +; NO_SVE-NEXT: stp q0, q4, [x0] +; NO_SVE-NEXT: ldp d11, d10, [sp], #32 // 16-byte Folded Reload +; NO_SVE-NEXT: ret +; ; VBITS_GE_2048-LABEL: select_v32f64: ; VBITS_GE_2048: // %bb.0: ; VBITS_GE_2048-NEXT: and w8, w2, #0x1 Index: llvm/test/CodeGen/AArch64/sve-fixed-length-fp-vselect.ll =================================================================== --- llvm/test/CodeGen/AArch64/sve-fixed-length-fp-vselect.ll +++ llvm/test/CodeGen/AArch64/sve-fixed-length-fp-vselect.ll @@ -99,7 +99,11 @@ ; CHECK-NEXT: sel z0.h, p1, z1.h, z2.h ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: mov sp, x29 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %mask = load <16 x i1>, <16 x i1>* %c %op1 = load <16 x half>, <16 x half>* %a @@ -195,7 +199,11 @@ ; VBITS_GE_512-NEXT: sel z0.h, p1, z1.h, z2.h ; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_512-NEXT: mov sp, x29 +; VBITS_GE_512-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_512-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_512-NEXT: .cfi_restore w30 +; VBITS_GE_512-NEXT: .cfi_restore w29 ; VBITS_GE_512-NEXT: ret %mask = load <32 x i1>, <32 x i1>* %c %op1 = load <32 x half>, <32 x half>* %a @@ -355,7 +363,11 @@ ; VBITS_GE_1024-NEXT: sel z0.h, p1, z1.h, z2.h ; VBITS_GE_1024-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_1024-NEXT: mov sp, x29 +; VBITS_GE_1024-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_1024-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_1024-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_1024-NEXT: .cfi_restore w30 +; VBITS_GE_1024-NEXT: .cfi_restore w29 ; VBITS_GE_1024-NEXT: ret %mask = load <64 x i1>, <64 x i1>* %c %op1 = load <64 x half>, <64 x half>* %a @@ -644,7 +656,11 @@ ; VBITS_GE_2048-NEXT: sel z0.h, p1, z1.h, z2.h ; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_2048-NEXT: mov sp, x29 +; VBITS_GE_2048-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_2048-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_2048-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_2048-NEXT: .cfi_restore w30 +; VBITS_GE_2048-NEXT: .cfi_restore w29 ; VBITS_GE_2048-NEXT: ret %mask = load <128 x i1>, <128 x i1>* %c %op1 = load <128 x half>, <128 x half>* %a @@ -713,7 +729,11 @@ ; CHECK-NEXT: sel z0.s, p1, z1.s, z2.s ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: mov sp, x29 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %mask = load <8 x i1>, <8 x i1>* %c %op1 = load <8 x float>, <8 x float>* %a @@ -769,7 +789,11 @@ ; VBITS_GE_512-NEXT: sel z0.s, p1, z1.s, z2.s ; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_512-NEXT: mov sp, x29 +; VBITS_GE_512-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_512-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_512-NEXT: .cfi_restore w30 +; VBITS_GE_512-NEXT: .cfi_restore w29 ; VBITS_GE_512-NEXT: ret %mask = load <16 x i1>, <16 x i1>* %c %op1 = load <16 x float>, <16 x float>* %a @@ -849,7 +873,11 @@ ; VBITS_GE_1024-NEXT: sel z0.s, p1, z1.s, z2.s ; VBITS_GE_1024-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_1024-NEXT: mov sp, x29 +; VBITS_GE_1024-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_1024-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_1024-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_1024-NEXT: .cfi_restore w30 +; VBITS_GE_1024-NEXT: .cfi_restore w29 ; VBITS_GE_1024-NEXT: ret %mask = load <32 x i1>, <32 x i1>* %c %op1 = load <32 x float>, <32 x float>* %a @@ -977,7 +1005,11 @@ ; VBITS_GE_2048-NEXT: sel z0.s, p1, z1.s, z2.s ; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_2048-NEXT: mov sp, x29 +; VBITS_GE_2048-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_2048-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_2048-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_2048-NEXT: .cfi_restore w30 +; VBITS_GE_2048-NEXT: .cfi_restore w29 ; VBITS_GE_2048-NEXT: ret %mask = load <64 x i1>, <64 x i1>* %c %op1 = load <64 x float>, <64 x float>* %a @@ -1044,7 +1076,11 @@ ; CHECK-NEXT: sel z0.d, p1, z1.d, z2.d ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: mov sp, x29 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %mask = load <4 x i1>, <4 x i1>* %c %op1 = load <4 x double>, <4 x double>* %a @@ -1095,7 +1131,11 @@ ; VBITS_GE_512-NEXT: sel z0.d, p1, z1.d, z2.d ; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0] ; VBITS_GE_512-NEXT: mov sp, x29 +; VBITS_GE_512-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_512-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_512-NEXT: .cfi_restore w30 +; VBITS_GE_512-NEXT: .cfi_restore w29 ; VBITS_GE_512-NEXT: ret %mask = load <8 x i1>, <8 x i1>* %c %op1 = load <8 x double>, <8 x double>* %a @@ -1166,7 +1206,11 @@ ; VBITS_GE_1024-NEXT: sel z0.d, p1, z1.d, z2.d ; VBITS_GE_1024-NEXT: st1d { z0.d }, p0, [x0] ; VBITS_GE_1024-NEXT: mov sp, x29 +; VBITS_GE_1024-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_1024-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_1024-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_1024-NEXT: .cfi_restore w30 +; VBITS_GE_1024-NEXT: .cfi_restore w29 ; VBITS_GE_1024-NEXT: ret %mask = load <16 x i1>, <16 x i1>* %c %op1 = load <16 x double>, <16 x double>* %a @@ -1308,7 +1352,11 @@ ; VBITS_GE_2048-NEXT: sel z0.d, p1, z1.d, z2.d ; VBITS_GE_2048-NEXT: st1d { z0.d }, p0, [x0] ; VBITS_GE_2048-NEXT: mov sp, x29 +; VBITS_GE_2048-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_2048-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_2048-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_2048-NEXT: .cfi_restore w30 +; VBITS_GE_2048-NEXT: .cfi_restore w29 ; VBITS_GE_2048-NEXT: ret %mask = load <32 x i1>, <32 x i1>* %c %op1 = load <32 x double>, <32 x double>* %a @@ -1318,4 +1366,4 @@ ret void } -attributes #0 = { "target-features"="+sve" } +attributes #0 = { "target-features"="+sve" uwtable } Index: llvm/test/CodeGen/AArch64/sve-fixed-length-int-vselect.ll =================================================================== --- llvm/test/CodeGen/AArch64/sve-fixed-length-int-vselect.ll +++ llvm/test/CodeGen/AArch64/sve-fixed-length-int-vselect.ll @@ -130,7 +130,11 @@ ; CHECK-NEXT: sel z0.b, p1, z1.b, z2.b ; CHECK-NEXT: st1b { z0.b }, p0, [x0] ; CHECK-NEXT: mov sp, x29 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %mask = load <32 x i1>, <32 x i1>* %c %op1 = load <32 x i8>, <32 x i8>* %a @@ -290,7 +294,11 @@ ; VBITS_GE_512-NEXT: sel z0.b, p1, z1.b, z2.b ; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0] ; VBITS_GE_512-NEXT: mov sp, x29 +; VBITS_GE_512-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_512-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_512-NEXT: .cfi_restore w30 +; VBITS_GE_512-NEXT: .cfi_restore w29 ; VBITS_GE_512-NEXT: ret %mask = load <64 x i1>, <64 x i1>* %c %op1 = load <64 x i8>, <64 x i8>* %a @@ -579,7 +587,11 @@ ; VBITS_GE_1024-NEXT: sel z0.b, p1, z1.b, z2.b ; VBITS_GE_1024-NEXT: st1b { z0.b }, p0, [x0] ; VBITS_GE_1024-NEXT: mov sp, x29 +; VBITS_GE_1024-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_1024-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_1024-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_1024-NEXT: .cfi_restore w30 +; VBITS_GE_1024-NEXT: .cfi_restore w29 ; VBITS_GE_1024-NEXT: ret %mask = load <128 x i1>, <128 x i1>* %c %op1 = load <128 x i8>, <128 x i8>* %a @@ -1126,7 +1138,11 @@ ; VBITS_GE_2048-NEXT: sel z0.b, p1, z1.b, z2.b ; VBITS_GE_2048-NEXT: st1b { z0.b }, p0, [x0] ; VBITS_GE_2048-NEXT: mov sp, x29 +; VBITS_GE_2048-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_2048-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_2048-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_2048-NEXT: .cfi_restore w30 +; VBITS_GE_2048-NEXT: .cfi_restore w29 ; VBITS_GE_2048-NEXT: ret %mask = load <256 x i1>, <256 x i1>* %c %op1 = load <256 x i8>, <256 x i8>* %a @@ -1215,7 +1231,11 @@ ; CHECK-NEXT: sel z0.h, p1, z1.h, z2.h ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: mov sp, x29 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %mask = load <16 x i1>, <16 x i1>* %c %op1 = load <16 x i16>, <16 x i16>* %a @@ -1311,7 +1331,11 @@ ; VBITS_GE_512-NEXT: sel z0.h, p1, z1.h, z2.h ; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_512-NEXT: mov sp, x29 +; VBITS_GE_512-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_512-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_512-NEXT: .cfi_restore w30 +; VBITS_GE_512-NEXT: .cfi_restore w29 ; VBITS_GE_512-NEXT: ret %mask = load <32 x i1>, <32 x i1>* %c %op1 = load <32 x i16>, <32 x i16>* %a @@ -1471,7 +1495,11 @@ ; VBITS_GE_1024-NEXT: sel z0.h, p1, z1.h, z2.h ; VBITS_GE_1024-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_1024-NEXT: mov sp, x29 +; VBITS_GE_1024-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_1024-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_1024-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_1024-NEXT: .cfi_restore w30 +; VBITS_GE_1024-NEXT: .cfi_restore w29 ; VBITS_GE_1024-NEXT: ret %mask = load <64 x i1>, <64 x i1>* %c %op1 = load <64 x i16>, <64 x i16>* %a @@ -1760,7 +1788,11 @@ ; VBITS_GE_2048-NEXT: sel z0.h, p1, z1.h, z2.h ; VBITS_GE_2048-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_2048-NEXT: mov sp, x29 +; VBITS_GE_2048-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_2048-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_2048-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_2048-NEXT: .cfi_restore w30 +; VBITS_GE_2048-NEXT: .cfi_restore w29 ; VBITS_GE_2048-NEXT: ret %mask = load <128 x i1>, <128 x i1>* %c %op1 = load <128 x i16>, <128 x i16>* %a @@ -1829,7 +1861,11 @@ ; CHECK-NEXT: sel z0.s, p1, z1.s, z2.s ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: mov sp, x29 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %mask = load <8 x i1>, <8 x i1>* %c %op1 = load <8 x i32>, <8 x i32>* %a @@ -1885,7 +1921,11 @@ ; VBITS_GE_512-NEXT: sel z0.s, p1, z1.s, z2.s ; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_512-NEXT: mov sp, x29 +; VBITS_GE_512-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_512-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_512-NEXT: .cfi_restore w30 +; VBITS_GE_512-NEXT: .cfi_restore w29 ; VBITS_GE_512-NEXT: ret %mask = load <16 x i1>, <16 x i1>* %c %op1 = load <16 x i32>, <16 x i32>* %a @@ -1965,7 +2005,11 @@ ; VBITS_GE_1024-NEXT: sel z0.s, p1, z1.s, z2.s ; VBITS_GE_1024-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_1024-NEXT: mov sp, x29 +; VBITS_GE_1024-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_1024-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_1024-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_1024-NEXT: .cfi_restore w30 +; VBITS_GE_1024-NEXT: .cfi_restore w29 ; VBITS_GE_1024-NEXT: ret %mask = load <32 x i1>, <32 x i1>* %c %op1 = load <32 x i32>, <32 x i32>* %a @@ -2093,7 +2137,11 @@ ; VBITS_GE_2048-NEXT: sel z0.s, p1, z1.s, z2.s ; VBITS_GE_2048-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_2048-NEXT: mov sp, x29 +; VBITS_GE_2048-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_2048-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_2048-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_2048-NEXT: .cfi_restore w30 +; VBITS_GE_2048-NEXT: .cfi_restore w29 ; VBITS_GE_2048-NEXT: ret %mask = load <64 x i1>, <64 x i1>* %c %op1 = load <64 x i32>, <64 x i32>* %a @@ -2160,7 +2208,11 @@ ; CHECK-NEXT: sel z0.d, p1, z1.d, z2.d ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: mov sp, x29 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %mask = load <4 x i1>, <4 x i1>* %c %op1 = load <4 x i64>, <4 x i64>* %a @@ -2211,7 +2263,11 @@ ; VBITS_GE_512-NEXT: sel z0.d, p1, z1.d, z2.d ; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0] ; VBITS_GE_512-NEXT: mov sp, x29 +; VBITS_GE_512-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_512-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_512-NEXT: .cfi_restore w30 +; VBITS_GE_512-NEXT: .cfi_restore w29 ; VBITS_GE_512-NEXT: ret %mask = load <8 x i1>, <8 x i1>* %c %op1 = load <8 x i64>, <8 x i64>* %a @@ -2282,7 +2338,11 @@ ; VBITS_GE_1024-NEXT: sel z0.d, p1, z1.d, z2.d ; VBITS_GE_1024-NEXT: st1d { z0.d }, p0, [x0] ; VBITS_GE_1024-NEXT: mov sp, x29 +; VBITS_GE_1024-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_1024-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_1024-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_1024-NEXT: .cfi_restore w30 +; VBITS_GE_1024-NEXT: .cfi_restore w29 ; VBITS_GE_1024-NEXT: ret %mask = load <16 x i1>, <16 x i1>* %c %op1 = load <16 x i64>, <16 x i64>* %a @@ -2424,7 +2484,11 @@ ; VBITS_GE_2048-NEXT: sel z0.d, p1, z1.d, z2.d ; VBITS_GE_2048-NEXT: st1d { z0.d }, p0, [x0] ; VBITS_GE_2048-NEXT: mov sp, x29 +; VBITS_GE_2048-NEXT: .cfi_def_cfa wsp, 16 ; VBITS_GE_2048-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; VBITS_GE_2048-NEXT: .cfi_def_cfa_offset 0 +; VBITS_GE_2048-NEXT: .cfi_restore w30 +; VBITS_GE_2048-NEXT: .cfi_restore w29 ; VBITS_GE_2048-NEXT: ret %mask = load <32 x i1>, <32 x i1>* %c %op1 = load <32 x i64>, <32 x i64>* %a @@ -2434,4 +2498,4 @@ ret void } -attributes #0 = { "target-features"="+sve" } +attributes #0 = { "target-features"="+sve" uwtable } Index: llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle.ll =================================================================== --- llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle.ll +++ llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle.ll @@ -950,7 +950,11 @@ ; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: mov sp, x29 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %op1 = load <4 x double>, <4 x double>* %a %op2 = load <4 x double>, <4 x double>* %b @@ -959,4 +963,4 @@ ret void } -attributes #0 = { "target-features"="+sve" } +attributes #0 = { "target-features"="+sve" uwtable } Index: llvm/test/CodeGen/AArch64/sve-insert-element.ll =================================================================== --- llvm/test/CodeGen/AArch64/sve-insert-element.ll +++ llvm/test/CodeGen/AArch64/sve-insert-element.ll @@ -496,7 +496,7 @@ ret %res } -define @test_predicate_insert_32xi1( %val, i1 %elt, i32 %idx) { +define @test_predicate_insert_32xi1( %val, i1 %elt, i32 %idx) uwtable { ; CHECK-LABEL: test_predicate_insert_32xi1: ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill @@ -524,7 +524,10 @@ ; CHECK-NEXT: cmpne p0.b, p1/z, z0.b, #0 ; CHECK-NEXT: cmpne p1.b, p1/z, z1.b, #0 ; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %res = insertelement %val, i1 %elt, i32 %idx ret %res Index: llvm/test/CodeGen/AArch64/sve-insert-vector.ll =================================================================== --- llvm/test/CodeGen/AArch64/sve-insert-vector.ll +++ llvm/test/CodeGen/AArch64/sve-insert-vector.ll @@ -206,7 +206,7 @@ ret void } -define void @insert_v2i64_nxv16i64(<2 x i64> %sv0, <2 x i64> %sv1, * %out) { +define void @insert_v2i64_nxv16i64(<2 x i64> %sv0, <2 x i64> %sv1, * %out) uwtable { ; CHECK-LABEL: insert_v2i64_nxv16i64: ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill @@ -227,7 +227,10 @@ ; CHECK-NEXT: st1d { z1.d }, p0, [x0, #1, mul vl] ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: addvl sp, sp, #4 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %v0 = call @llvm.experimental.vector.insert.v2i64.nxv16i64( undef, <2 x i64> %sv0, i64 0) %v = call @llvm.experimental.vector.insert.v2i64.nxv16i64( %v0, <2 x i64> %sv1, i64 4) @@ -248,7 +251,7 @@ ret void } -define void @insert_v2i64_nxv16i64_lo2(<2 x i64>* %psv, * %out) { +define void @insert_v2i64_nxv16i64_lo2(<2 x i64>* %psv, * %out) uwtable { ; CHECK-LABEL: insert_v2i64_nxv16i64_lo2: ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill @@ -264,7 +267,10 @@ ; CHECK-NEXT: st1d { z1.d }, p0, [x1, #1, mul vl] ; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %sv = load <2 x i64>, <2 x i64>* %psv %v = call @llvm.experimental.vector.insert.v2i64.nxv16i64( undef, <2 x i64> %sv, i64 2) Index: llvm/test/CodeGen/AArch64/sve-ldnf1.mir =================================================================== --- llvm/test/CodeGen/AArch64/sve-ldnf1.mir +++ llvm/test/CodeGen/AArch64/sve-ldnf1.mir @@ -4,26 +4,26 @@ # Test that prologepilog works for each of the LDNF1 instructions for stack-based objects. # --- | - define void @testcase_positive_offset() { + define void @testcase_positive_offset() uwtable { %dummy = alloca , align 8 %object = alloca , align 8 ; Reads from %object at offset 63 * readsize ret void } - define void @testcase_negative_offset() { + define void @testcase_negative_offset() uwtable { %dummy = alloca , align 8 %object = alloca , align 8 ; Reads from %object at offset 63 * readsize ret void } - define void @testcase_positive_offset_out_of_range() { + define void @testcase_positive_offset_out_of_range() uwtable { %dummy = alloca , align 8 %object = alloca , align 8 ; Reads from %object at offset 64 * readsize ret void } - define void @testcase_negative_offset_out_of_range() { + define void @testcase_negative_offset_out_of_range() uwtable { %dummy = alloca , align 8 %object = alloca , align 8 ; Reads from %object at offset -1 * readsize @@ -65,7 +65,10 @@ ; CHECK-NEXT: renamable $z0 = LDNF1SW_D_IMM renamable $p0, $sp, 7 :: (load (s32) from %ir.object, align 8) ; CHECK-NEXT: renamable $z0 = LDNF1D_IMM renamable $p0, $sp, 7 :: (load (s64) from %ir.object) ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16 ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2) + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29 ; CHECK-NEXT: RET_ReallyLR implicit $z0 renamable $z0 = LDNF1B_IMM renamable $p0, %stack.1.object, 7 :: (load 1 from %ir.object, align 2) renamable $z0 = LDNF1B_H_IMM renamable $p0, %stack.1.object, 7 :: (load 1 from %ir.object, align 2) @@ -121,7 +124,10 @@ ; CHECK-NEXT: renamable $z0 = LDNF1SW_D_IMM renamable $p0, $sp, -8 :: (load (s32) from %ir.object, align 8) ; CHECK-NEXT: renamable $z0 = LDNF1D_IMM renamable $p0, $sp, -8 :: (load (s64) from %ir.object) ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16 ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2) + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29 ; CHECK-NEXT: RET_ReallyLR implicit $z0 renamable $z0 = LDNF1B_IMM renamable $p0, %stack.1.object, -8 :: (load 1 from %ir.object, align 2) renamable $z0 = LDNF1B_H_IMM renamable $p0, %stack.1.object, -8 :: (load 1 from %ir.object, align 2) @@ -191,7 +197,10 @@ ; CHECK-NEXT: $x8 = ADDPL_XXI $sp, 4 ; CHECK-NEXT: renamable $z0 = LDNF1SW_D_IMM renamable $p0, killed $x8, 7 :: (load (s32) from %ir.object, align 8) ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16 ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2) + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29 ; CHECK-NEXT: RET_ReallyLR implicit $z0 renamable $z0 = LDNF1B_IMM renamable $p0, %stack.1.object, 8 :: (load 1 from %ir.object, align 2) renamable $z0 = LDNF1B_H_IMM renamable $p0, %stack.1.object, 8 :: (load 1 from %ir.object, align 2) @@ -260,7 +269,10 @@ ; CHECK-NEXT: $x8 = ADDPL_XXI $sp, -4 ; CHECK-NEXT: renamable $z0 = LDNF1SW_D_IMM renamable $p0, killed $x8, -8 :: (load (s32) from %ir.object, align 8) ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16 ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2) + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29 ; CHECK-NEXT: RET_ReallyLR implicit $z0 renamable $z0 = LDNF1B_IMM renamable $p0, %stack.1.object, -9 :: (load 1 from %ir.object, align 2) renamable $z0 = LDNF1B_H_IMM renamable $p0, %stack.1.object, -9 :: (load 1 from %ir.object, align 2) Index: llvm/test/CodeGen/AArch64/sve-ldstnt1.mir =================================================================== --- llvm/test/CodeGen/AArch64/sve-ldstnt1.mir +++ llvm/test/CodeGen/AArch64/sve-ldstnt1.mir @@ -4,26 +4,26 @@ # Test that prologepilog works for each of the LDNT1/STNT1 instructions for stack-based objects. # --- | - define void @testcase_positive_offset() { + define void @testcase_positive_offset() uwtable { %dummy = alloca , align 8 %object = alloca , align 8 ; Reads from %object at offset 7 * readsize ret void } - define void @testcase_negative_offset() { + define void @testcase_negative_offset() uwtable { %dummy = alloca , align 8 %object = alloca , align 8 ; Reads from %object at offset -8 * readsize ret void } - define void @testcase_positive_offset_out_of_range() { + define void @testcase_positive_offset_out_of_range() uwtable { %dummy = alloca , align 8 %object = alloca , align 8 ; Reads from %object at offset 8 * readsize ret void } - define void @testcase_negative_offset_out_of_range() { + define void @testcase_negative_offset_out_of_range() uwtable { %dummy = alloca , align 8 %object = alloca , align 8 ; Reads from %object at offset -9 * readsize @@ -57,7 +57,10 @@ ; CHECK-NEXT: STNT1W_ZRI renamable $z0, renamable $p0, $sp, 7 :: (store (s32) into %ir.object, align 8) ; CHECK-NEXT: STNT1D_ZRI renamable $z0, renamable $p0, $sp, 7 :: (store (s64) into %ir.object) ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16 ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2) + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29 ; CHECK-NEXT: RET_ReallyLR implicit $z0 renamable $z0 = LDNT1B_ZRI renamable $p0, %stack.1.object, 7 :: (load 1 from %ir.object, align 2) renamable $z0 = LDNT1H_ZRI renamable $p0, %stack.1.object, 7 :: (load 2 from %ir.object, align 2) @@ -97,7 +100,10 @@ ; CHECK-NEXT: STNT1W_ZRI renamable $z0, renamable $p0, $sp, -8 :: (store (s32) into %ir.object, align 8) ; CHECK-NEXT: STNT1D_ZRI renamable $z0, renamable $p0, $sp, -8 :: (store (s64) into %ir.object) ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16 ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2) + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29 ; CHECK-NEXT: RET_ReallyLR implicit $z0 renamable $z0 = LDNT1B_ZRI renamable $p0, %stack.1.object, -8 :: (load 1 from %ir.object, align 2) renamable $z0 = LDNT1H_ZRI renamable $p0, %stack.1.object, -8 :: (load 2 from %ir.object, align 2) @@ -145,7 +151,10 @@ ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, 1 ; CHECK-NEXT: STNT1D_ZRI renamable $z0, renamable $p0, killed $x8, 7 :: (store (s64) into %ir.object) ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16 ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2) + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29 ; CHECK-NEXT: RET_ReallyLR implicit $z0 renamable $z0 = LDNT1B_ZRI renamable $p0, %stack.1.object, 8 :: (load 1 from %ir.object, align 2) renamable $z0 = LDNT1H_ZRI renamable $p0, %stack.1.object, 8 :: (load 2 from %ir.object, align 2) @@ -193,7 +202,10 @@ ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, -1 ; CHECK-NEXT: STNT1D_ZRI renamable $z0, renamable $p0, killed $x8, -8 :: (store (s64) into %ir.object) ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16 ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2) + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0 + ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29 ; CHECK-NEXT: RET_ReallyLR implicit $z0 renamable $z0 = LDNT1B_ZRI renamable $p0, %stack.1.object, -9 :: (load 1 from %ir.object, align 2) renamable $z0 = LDNT1H_ZRI renamable $p0, %stack.1.object, -9 :: (load 2 from %ir.object, align 2) Index: llvm/test/CodeGen/AArch64/sve-pred-arith.ll =================================================================== --- llvm/test/CodeGen/AArch64/sve-pred-arith.ll +++ llvm/test/CodeGen/AArch64/sve-pred-arith.ll @@ -46,7 +46,7 @@ ; ILLEGAL ADDS -define aarch64_sve_vector_pcs @add_nxv64i1( %a, %b) { +define aarch64_sve_vector_pcs @add_nxv64i1( %a, %b) uwtable { ; CHECK-LABEL: add_nxv64i1: ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill @@ -74,7 +74,10 @@ ; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %res = add %a, %b ret %res; @@ -127,7 +130,7 @@ ; ILLEGAL SUBGS -define aarch64_sve_vector_pcs @sub_nxv64i1( %a, %b) { +define aarch64_sve_vector_pcs @sub_nxv64i1( %a, %b) uwtable { ; CHECK-LABEL: sub_nxv64i1: ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill @@ -155,7 +158,10 @@ ; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %res = sub %a, %b ret %res; Index: llvm/test/CodeGen/AArch64/sve-varargs.ll =================================================================== --- llvm/test/CodeGen/AArch64/sve-varargs.ll +++ llvm/test/CodeGen/AArch64/sve-varargs.ll @@ -5,7 +5,7 @@ @.str_1 = internal constant [6 x i8] c"boo!\0A\00" -define void @foo( %x) { +define void @foo( %x) uwtable { ; CHECK-LABEL: foo: ; CHECK: // %bb.0: ; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill @@ -15,6 +15,8 @@ ; CHECK-NEXT: add x0, x0, :lo12:.str_1 ; CHECK-NEXT: bl sve_printf ; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret %f = getelementptr [6 x i8], [6 x i8]* @.str_1, i64 0, i64 0 call i32 (i8*, , ...) @sve_printf(i8* %f, %x) Index: llvm/test/CodeGen/AArch64/swifttail-call.ll =================================================================== --- llvm/test/CodeGen/AArch64/swifttail-call.ll +++ llvm/test/CodeGen/AArch64/swifttail-call.ll @@ -16,17 +16,18 @@ ; COMMON-NEXT: b callee_stack0 } -define swifttailcc void @caller_to0_from8([8 x i64], i64) { +define swifttailcc void @caller_to0_from8([8 x i64], i64) #0 { ; COMMON-LABEL: caller_to0_from8: musttail call swifttailcc void @callee_stack0() ret void ; COMMON: add sp, sp, #16 +; COMMON-NEXT: .cfi_def_cfa_offset -16 ; COMMON-NEXT: b callee_stack0 } -define swifttailcc void @caller_to8_from0() { +define swifttailcc void @caller_to8_from0() #0 { ; COMMON-LABEL: caller_to8_from0: ; Key point is that the "42" should go #16 below incoming stack @@ -38,7 +39,7 @@ ; COMMON-NEXT: b callee_stack8 } -define swifttailcc void @caller_to8_from8([8 x i64], i64 %a) { +define swifttailcc void @caller_to8_from8([8 x i64], i64 %a) #0 { ; COMMON-LABEL: caller_to8_from8: ; COMMON-NOT: sub sp, @@ -50,7 +51,7 @@ ; COMMON-NEXT: b callee_stack8 } -define swifttailcc void @caller_to16_from8([8 x i64], i64 %a) { +define swifttailcc void @caller_to16_from8([8 x i64], i64 %a) #0 { ; COMMON-LABEL: caller_to16_from8: ; COMMON-NOT: sub sp, @@ -65,7 +66,7 @@ } -define swifttailcc void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) { +define swifttailcc void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) #0 { ; COMMON-LABEL: caller_to8_from24: ; COMMON-NOT: sub sp, @@ -74,11 +75,12 @@ ret void ; COMMON: str {{x[0-9]+}}, [sp, #16]! +; COMMON-NEXT: .cfi_def_cfa_offset -16 ; COMMON-NEXT: b callee_stack8 } -define swifttailcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) { +define swifttailcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) #0 { ; COMMON-LABEL: caller_to16_from16: ; COMMON-NOT: sub sp, @@ -105,7 +107,7 @@ ; Weakly-referenced extern functions cannot be tail-called, as AAELF does ; not define the behaviour of branch instructions to undefined weak symbols. -define swifttailcc void @caller_weak() { +define swifttailcc void @caller_weak() #0 { ; COMMON-LABEL: caller_weak: ; COMMON: bl callee_weak tail call void @callee_weak() @@ -114,7 +116,7 @@ declare { [2 x float] } @get_vec2() -define { [3 x float] } @test_add_elem() { +define { [3 x float] } @test_add_elem() #0 { ; SDAG-LABEL: test_add_elem: ; SDAG: bl get_vec2 ; SDAG: fmov s2, #1.0 @@ -138,7 +140,7 @@ } declare double @get_double() -define { double, [2 x double] } @test_mismatched_insert() { +define { double, [2 x double] } @test_mismatched_insert() #0 { ; COMMON-LABEL: test_mismatched_insert: ; COMMON: bl get_double ; COMMON: bl get_double @@ -156,7 +158,7 @@ ret { double, [2 x double] } %res.012 } -define void @fromC_totail() { +define void @fromC_totail() #0 { ; COMMON-LABEL: fromC_totail: ; COMMON: sub sp, sp, #48 @@ -174,7 +176,7 @@ ret void } -define void @fromC_totail_noreservedframe(i32 %len) { +define void @fromC_totail_noreservedframe(i32 %len) #0 { ; COMMON-LABEL: fromC_totail_noreservedframe: ; COMMON: stp x29, x30, [sp, #-48]! @@ -198,7 +200,7 @@ declare void @Ccallee_stack8([8 x i64], i64) -define swifttailcc void @fromtail_toC() { +define swifttailcc void @fromtail_toC() #0 { ; COMMON-LABEL: fromtail_toC: ; COMMON: sub sp, sp, #32 @@ -220,7 +222,7 @@ } declare swifttailcc i8* @SwiftSelf(i8 * swiftasync %context, i8* swiftself %closure) -define swiftcc i8* @CallSwiftSelf(i8* swiftself %closure, i8* %context) { +define swiftcc i8* @CallSwiftSelf(i8* swiftself %closure, i8* %context) #0 { ; CHECK-LABEL: CallSwiftSelf: ; CHECK: stp x20 ;call void asm "","~{r13}"() ; We get a push r13 but why not with the call @@ -228,3 +230,5 @@ %res = call swifttailcc i8* @SwiftSelf(i8 * swiftasync %context, i8* swiftself %closure) ret i8* %res } + +attributes #0 = { uwtable } \ No newline at end of file Index: llvm/test/CodeGen/AArch64/tail-call.ll =================================================================== --- llvm/test/CodeGen/AArch64/tail-call.ll +++ llvm/test/CodeGen/AArch64/tail-call.ll @@ -16,17 +16,18 @@ ; COMMON-NEXT: b callee_stack0 } -define fastcc void @caller_to0_from8([8 x i64], i64) { +define fastcc void @caller_to0_from8([8 x i64], i64) #0 { ; COMMON-LABEL: caller_to0_from8: tail call fastcc void @callee_stack0() ret void ; COMMON: add sp, sp, #16 +; COMMON: .cfi_def_cfa_offset -16 ; COMMON-NEXT: b callee_stack0 } -define fastcc void @caller_to8_from0() { +define fastcc void @caller_to8_from0() #0 { ; COMMON-LABEL: caller_to8_from0: ; Key point is that the "42" should go #16 below incoming stack @@ -38,7 +39,7 @@ ; COMMON-NEXT: b callee_stack8 } -define fastcc void @caller_to8_from8([8 x i64], i64 %a) { +define fastcc void @caller_to8_from8([8 x i64], i64 %a) #0 { ; COMMON-LABEL: caller_to8_from8: ; COMMON-NOT: sub sp, @@ -50,7 +51,7 @@ ; COMMON-NEXT: b callee_stack8 } -define fastcc void @caller_to16_from8([8 x i64], i64 %a) { +define fastcc void @caller_to16_from8([8 x i64], i64 %a) #0 { ; COMMON-LABEL: caller_to16_from8: ; COMMON-NOT: sub sp, @@ -65,7 +66,7 @@ } -define fastcc void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) { +define fastcc void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) #0 { ; COMMON-LABEL: caller_to8_from24: ; COMMON-NOT: sub sp, @@ -74,11 +75,12 @@ ret void ; COMMON: str {{x[0-9]+}}, [sp, #16]! +; COMMON: .cfi_def_cfa_offset -16 ; COMMON-NEXT: b callee_stack8 } -define fastcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) { +define fastcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) #0 { ; COMMON-LABEL: caller_to16_from16: ; COMMON-NOT: sub sp, @@ -105,7 +107,7 @@ ; Weakly-referenced extern functions cannot be tail-called, as AAELF does ; not define the behaviour of branch instructions to undefined weak symbols. -define fastcc void @caller_weak() { +define fastcc void @caller_weak() #0 { ; COMMON-LABEL: caller_weak: ; COMMON: bl callee_weak tail call void @callee_weak() @@ -114,7 +116,7 @@ declare { [2 x float] } @get_vec2() -define { [3 x float] } @test_add_elem() { +define { [3 x float] } @test_add_elem() #0 { ; SDAG-LABEL: test_add_elem: ; SDAG: bl get_vec2 ; SDAG: fmov s2, #1.0 @@ -138,7 +140,7 @@ } declare double @get_double() -define { double, [2 x double] } @test_mismatched_insert() { +define { double, [2 x double] } @test_mismatched_insert() #0 { ; COMMON-LABEL: test_mismatched_insert: ; COMMON: bl get_double ; COMMON: bl get_double @@ -155,3 +157,5 @@ ret { double, [2 x double] } %res.012 } + +attributes #0 = { uwtable } \ No newline at end of file Index: llvm/test/CodeGen/AArch64/tailcc-tail-call.ll =================================================================== --- llvm/test/CodeGen/AArch64/tailcc-tail-call.ll +++ llvm/test/CodeGen/AArch64/tailcc-tail-call.ll @@ -16,17 +16,18 @@ ; COMMON-NEXT: b callee_stack0 } -define tailcc void @caller_to0_from8([8 x i64], i64) { +define tailcc void @caller_to0_from8([8 x i64], i64) #0 { ; COMMON-LABEL: caller_to0_from8: tail call tailcc void @callee_stack0() ret void ; COMMON: add sp, sp, #16 +; COMMON: .cfi_def_cfa_offset -16 ; COMMON-NEXT: b callee_stack0 } -define tailcc void @caller_to8_from0() "frame-pointer"="all"{ +define tailcc void @caller_to8_from0() "frame-pointer"="all" uwtable { ; COMMON-LABEL: caller_to8_from0: ; Key point is that the "42" should go #16 below incoming stack @@ -40,10 +41,13 @@ ; from an interrupt if the kernel does not honour a red-zone, and a larger ; call could well overflow the red zone even if it is present. ; COMMON-NOT: sub sp, +; COMMON-NEXT: .cfi_def_cfa_offset 16 +; COMMON-NEXT: .cfi_restore w30 +; COMMON-NEXT: .cfi_restore w29 ; COMMON-NEXT: b callee_stack8 } -define tailcc void @caller_to8_from8([8 x i64], i64 %a) { +define tailcc void @caller_to8_from8([8 x i64], i64 %a) #0 { ; COMMON-LABEL: caller_to8_from8: ; COMMON-NOT: sub sp, @@ -55,7 +59,7 @@ ; COMMON-NEXT: b callee_stack8 } -define tailcc void @caller_to16_from8([8 x i64], i64 %a) { +define tailcc void @caller_to16_from8([8 x i64], i64 %a) #0 { ; COMMON-LABEL: caller_to16_from8: ; COMMON-NOT: sub sp, @@ -70,7 +74,7 @@ } -define tailcc void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) { +define tailcc void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) #0 { ; COMMON-LABEL: caller_to8_from24: ; COMMON-NOT: sub sp, @@ -79,11 +83,12 @@ ret void ; COMMON: str {{x[0-9]+}}, [sp, #16]! +; COMMON-NEXT: .cfi_def_cfa_offset -16 ; COMMON-NEXT: b callee_stack8 } -define tailcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) { +define tailcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) #0 { ; COMMON-LABEL: caller_to16_from16: ; COMMON-NOT: sub sp, @@ -110,7 +115,7 @@ ; Weakly-referenced extern functions cannot be tail-called, as AAELF does ; not define the behaviour of branch instructions to undefined weak symbols. -define tailcc void @caller_weak() { +define tailcc void @caller_weak() #0 { ; COMMON-LABEL: caller_weak: ; COMMON: bl callee_weak tail call void @callee_weak() @@ -119,7 +124,7 @@ declare { [2 x float] } @get_vec2() -define { [3 x float] } @test_add_elem() { +define { [3 x float] } @test_add_elem() #0 { ; SDAG-LABEL: test_add_elem: ; SDAG: bl get_vec2 ; SDAG: fmov s2, #1.0 @@ -143,7 +148,7 @@ } declare double @get_double() -define { double, [2 x double] } @test_mismatched_insert() { +define { double, [2 x double] } @test_mismatched_insert() #0 { ; COMMON-LABEL: test_mismatched_insert: ; COMMON: bl get_double ; COMMON: bl get_double @@ -161,7 +166,7 @@ ret { double, [2 x double] } %res.012 } -define void @fromC_totail() { +define void @fromC_totail() #0 { ; COMMON-LABEL: fromC_totail: ; COMMON: sub sp, sp, #32 @@ -179,7 +184,7 @@ ret void } -define void @fromC_totail_noreservedframe(i32 %len) { +define void @fromC_totail_noreservedframe(i32 %len) #0 { ; COMMON-LABEL: fromC_totail_noreservedframe: ; COMMON: stp x29, x30, [sp, #-32]! @@ -203,7 +208,7 @@ declare void @Ccallee_stack8([8 x i64], i64) -define tailcc void @fromtail_toC() { +define tailcc void @fromtail_toC() #0 { ; COMMON-LABEL: fromtail_toC: ; COMMON: sub sp, sp, #32 @@ -223,3 +228,5 @@ call void @Ccallee_stack8([8 x i64] undef, i64 42) ret void } + +attributes #0 = { uwtable } \ No newline at end of file Index: llvm/test/CodeGen/AArch64/unwind-preserved.ll =================================================================== --- llvm/test/CodeGen/AArch64/unwind-preserved.ll +++ llvm/test/CodeGen/AArch64/unwind-preserved.ll @@ -3,7 +3,7 @@ ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -O0 -global-isel=1 -global-isel-abort=0 < %s | FileCheck %s --check-prefix=GISEL ; Test that z0 is saved/restored, as the unwinder may only retain the low 64bits (d0). -define @invoke_callee_may_throw_sve( %v) personality i8 0 { +define @invoke_callee_may_throw_sve( %v) uwtable personality i8 0 { ; CHECK-LABEL: invoke_callee_may_throw_sve: ; CHECK: .Lfunc_begin0: ; CHECK-NEXT: .cfi_startproc @@ -61,6 +61,7 @@ ; CHECK-NEXT: .LBB0_1: // %.Lcontinue ; CHECK-NEXT: ldr z0, [sp, #1, mul vl] // 16-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG ; CHECK-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload @@ -90,12 +91,25 @@ ; CHECK-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload ; CHECK-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #18 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 +; CHECK-NEXT: .cfi_restore z8 +; CHECK-NEXT: .cfi_restore z9 +; CHECK-NEXT: .cfi_restore z10 +; CHECK-NEXT: .cfi_restore z11 +; CHECK-NEXT: .cfi_restore z12 +; CHECK-NEXT: .cfi_restore z13 +; CHECK-NEXT: .cfi_restore z14 +; CHECK-NEXT: .cfi_restore z15 ; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret ; CHECK-NEXT: .LBB0_2: // %.Lunwind ; CHECK-NEXT: .Ltmp2: ; CHECK-NEXT: ldr z0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG ; CHECK-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload @@ -125,7 +139,19 @@ ; CHECK-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload ; CHECK-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #18 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 +; CHECK-NEXT: .cfi_restore z8 +; CHECK-NEXT: .cfi_restore z9 +; CHECK-NEXT: .cfi_restore z10 +; CHECK-NEXT: .cfi_restore z11 +; CHECK-NEXT: .cfi_restore z12 +; CHECK-NEXT: .cfi_restore z13 +; CHECK-NEXT: .cfi_restore z14 +; CHECK-NEXT: .cfi_restore z15 ; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret ; ; GISEL-LABEL: invoke_callee_may_throw_sve: @@ -185,6 +211,7 @@ ; GISEL-NEXT: .LBB0_1: // %.Lcontinue ; GISEL-NEXT: ldr z0, [sp, #1, mul vl] // 16-byte Folded Reload ; GISEL-NEXT: addvl sp, sp, #2 +; GISEL-NEXT: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG ; GISEL-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload ; GISEL-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload ; GISEL-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload @@ -214,12 +241,25 @@ ; GISEL-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload ; GISEL-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload ; GISEL-NEXT: addvl sp, sp, #18 +; GISEL-NEXT: .cfi_def_cfa wsp, 16 +; GISEL-NEXT: .cfi_restore z8 +; GISEL-NEXT: .cfi_restore z9 +; GISEL-NEXT: .cfi_restore z10 +; GISEL-NEXT: .cfi_restore z11 +; GISEL-NEXT: .cfi_restore z12 +; GISEL-NEXT: .cfi_restore z13 +; GISEL-NEXT: .cfi_restore z14 +; GISEL-NEXT: .cfi_restore z15 ; GISEL-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; GISEL-NEXT: .cfi_def_cfa_offset 0 +; GISEL-NEXT: .cfi_restore w30 +; GISEL-NEXT: .cfi_restore w29 ; GISEL-NEXT: ret ; GISEL-NEXT: .LBB0_2: // %.Lunwind ; GISEL-NEXT: .Ltmp2: ; GISEL-NEXT: ldr z0, [sp] // 16-byte Folded Reload ; GISEL-NEXT: addvl sp, sp, #2 +; GISEL-NEXT: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG ; GISEL-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload ; GISEL-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload ; GISEL-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload @@ -249,7 +289,19 @@ ; GISEL-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload ; GISEL-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload ; GISEL-NEXT: addvl sp, sp, #18 +; GISEL-NEXT: .cfi_def_cfa wsp, 16 +; GISEL-NEXT: .cfi_restore z8 +; GISEL-NEXT: .cfi_restore z9 +; GISEL-NEXT: .cfi_restore z10 +; GISEL-NEXT: .cfi_restore z11 +; GISEL-NEXT: .cfi_restore z12 +; GISEL-NEXT: .cfi_restore z13 +; GISEL-NEXT: .cfi_restore z14 +; GISEL-NEXT: .cfi_restore z15 ; GISEL-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload +; GISEL-NEXT: .cfi_def_cfa_offset 0 +; GISEL-NEXT: .cfi_restore w30 +; GISEL-NEXT: .cfi_restore w29 ; GISEL-NEXT: ret %result = invoke @may_throw_sve( %v) to label %.Lcontinue unwind label %.Lunwind .Lcontinue: @@ -263,7 +315,7 @@ ; Test that q0 is saved/restored, as the unwinder may only retain the low 64bits (d0). -define aarch64_vector_pcs <4 x i32> @invoke_callee_may_throw_neon(<4 x i32> %v) personality i8 0 { +define aarch64_vector_pcs <4 x i32> @invoke_callee_may_throw_neon(<4 x i32> %v) uwtable personality i8 0 { ; CHECK-LABEL: invoke_callee_may_throw_neon: ; CHECK: .Lfunc_begin1: ; CHECK-NEXT: .cfi_startproc @@ -315,6 +367,25 @@ ; CHECK-NEXT: ldp q21, q20, [sp, #64] // 32-byte Folded Reload ; CHECK-NEXT: ldp q23, q22, [sp, #32] // 32-byte Folded Reload ; CHECK-NEXT: add sp, sp, #304 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 +; CHECK-NEXT: .cfi_restore b8 +; CHECK-NEXT: .cfi_restore b9 +; CHECK-NEXT: .cfi_restore b10 +; CHECK-NEXT: .cfi_restore b11 +; CHECK-NEXT: .cfi_restore b12 +; CHECK-NEXT: .cfi_restore b13 +; CHECK-NEXT: .cfi_restore b14 +; CHECK-NEXT: .cfi_restore b15 +; CHECK-NEXT: .cfi_restore b16 +; CHECK-NEXT: .cfi_restore b17 +; CHECK-NEXT: .cfi_restore b18 +; CHECK-NEXT: .cfi_restore b19 +; CHECK-NEXT: .cfi_restore b20 +; CHECK-NEXT: .cfi_restore b21 +; CHECK-NEXT: .cfi_restore b22 +; CHECK-NEXT: .cfi_restore b23 ; CHECK-NEXT: ret ; CHECK-NEXT: .LBB1_2: // %.Lunwind ; CHECK-NEXT: .Ltmp5: @@ -329,6 +400,25 @@ ; CHECK-NEXT: ldp q21, q20, [sp, #64] // 32-byte Folded Reload ; CHECK-NEXT: ldp q23, q22, [sp, #32] // 32-byte Folded Reload ; CHECK-NEXT: add sp, sp, #304 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 +; CHECK-NEXT: .cfi_restore b8 +; CHECK-NEXT: .cfi_restore b9 +; CHECK-NEXT: .cfi_restore b10 +; CHECK-NEXT: .cfi_restore b11 +; CHECK-NEXT: .cfi_restore b12 +; CHECK-NEXT: .cfi_restore b13 +; CHECK-NEXT: .cfi_restore b14 +; CHECK-NEXT: .cfi_restore b15 +; CHECK-NEXT: .cfi_restore b16 +; CHECK-NEXT: .cfi_restore b17 +; CHECK-NEXT: .cfi_restore b18 +; CHECK-NEXT: .cfi_restore b19 +; CHECK-NEXT: .cfi_restore b20 +; CHECK-NEXT: .cfi_restore b21 +; CHECK-NEXT: .cfi_restore b22 +; CHECK-NEXT: .cfi_restore b23 ; CHECK-NEXT: ret ; ; GISEL-LABEL: invoke_callee_may_throw_neon: @@ -382,6 +472,25 @@ ; GISEL-NEXT: ldp q21, q20, [sp, #64] // 32-byte Folded Reload ; GISEL-NEXT: ldp q23, q22, [sp, #32] // 32-byte Folded Reload ; GISEL-NEXT: add sp, sp, #304 +; GISEL-NEXT: .cfi_def_cfa_offset 0 +; GISEL-NEXT: .cfi_restore w30 +; GISEL-NEXT: .cfi_restore w29 +; GISEL-NEXT: .cfi_restore b8 +; GISEL-NEXT: .cfi_restore b9 +; GISEL-NEXT: .cfi_restore b10 +; GISEL-NEXT: .cfi_restore b11 +; GISEL-NEXT: .cfi_restore b12 +; GISEL-NEXT: .cfi_restore b13 +; GISEL-NEXT: .cfi_restore b14 +; GISEL-NEXT: .cfi_restore b15 +; GISEL-NEXT: .cfi_restore b16 +; GISEL-NEXT: .cfi_restore b17 +; GISEL-NEXT: .cfi_restore b18 +; GISEL-NEXT: .cfi_restore b19 +; GISEL-NEXT: .cfi_restore b20 +; GISEL-NEXT: .cfi_restore b21 +; GISEL-NEXT: .cfi_restore b22 +; GISEL-NEXT: .cfi_restore b23 ; GISEL-NEXT: ret ; GISEL-NEXT: .LBB1_2: // %.Lunwind ; GISEL-NEXT: .Ltmp5: @@ -396,6 +505,25 @@ ; GISEL-NEXT: ldp q21, q20, [sp, #64] // 32-byte Folded Reload ; GISEL-NEXT: ldp q23, q22, [sp, #32] // 32-byte Folded Reload ; GISEL-NEXT: add sp, sp, #304 +; GISEL-NEXT: .cfi_def_cfa_offset 0 +; GISEL-NEXT: .cfi_restore w30 +; GISEL-NEXT: .cfi_restore w29 +; GISEL-NEXT: .cfi_restore b8 +; GISEL-NEXT: .cfi_restore b9 +; GISEL-NEXT: .cfi_restore b10 +; GISEL-NEXT: .cfi_restore b11 +; GISEL-NEXT: .cfi_restore b12 +; GISEL-NEXT: .cfi_restore b13 +; GISEL-NEXT: .cfi_restore b14 +; GISEL-NEXT: .cfi_restore b15 +; GISEL-NEXT: .cfi_restore b16 +; GISEL-NEXT: .cfi_restore b17 +; GISEL-NEXT: .cfi_restore b18 +; GISEL-NEXT: .cfi_restore b19 +; GISEL-NEXT: .cfi_restore b20 +; GISEL-NEXT: .cfi_restore b21 +; GISEL-NEXT: .cfi_restore b22 +; GISEL-NEXT: .cfi_restore b23 ; GISEL-NEXT: ret %result = invoke aarch64_vector_pcs <4 x i32> @may_throw_neon(<4 x i32> %v) to label %.Lcontinue unwind label %.Lunwind .Lcontinue: Index: llvm/test/Transforms/CodeGenPrepare/AArch64/large-offset-gep.ll =================================================================== --- llvm/test/Transforms/CodeGenPrepare/AArch64/large-offset-gep.ll +++ llvm/test/Transforms/CodeGenPrepare/AArch64/large-offset-gep.ll @@ -134,7 +134,7 @@ declare %struct_type* @foo() declare void @foo2() -define void @test4(i32 %n) personality i32 (...)* @__FrameHandler { +define void @test4(i32 %n) uwtable personality i32 (...)* @__FrameHandler { ; CHECK-LABEL: test4: ; CHECK: .Lfunc_begin0: ; CHECK-NEXT: .cfi_startproc @@ -171,6 +171,11 @@ ; CHECK-NEXT: .LBB3_4: // %while_end ; CHECK-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w19 +; CHECK-NEXT: .cfi_restore w20 +; CHECK-NEXT: .cfi_restore w21 +; CHECK-NEXT: .cfi_restore w30 ; CHECK-NEXT: ret ; CHECK-NEXT: .LBB3_5: // %cleanup ; CHECK-NEXT: .Ltmp2: Index: llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/aarch64_generated_funcs.ll.generated.expected =================================================================== --- llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/aarch64_generated_funcs.ll.generated.expected +++ llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/aarch64_generated_funcs.ll.generated.expected @@ -91,8 +91,12 @@ ; CHECK-NEXT: bl OUTLINED_FUNCTION_0 ; CHECK-NEXT: .LBB0_5: ; CHECK-NEXT: mov w0, wzr +; CHECK-NEXT: .cfi_def_cfa wsp, 48 ; CHECK-NEXT: ldp x29, x30, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #48 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret ; ; CHECK-LABEL: main: @@ -118,8 +122,12 @@ ; CHECK-NEXT: //NO_APP ; CHECK-NEXT: stp w10, w8, [x29, #-12] ; CHECK-NEXT: stp w9, w11, [sp, #12] +; CHECK-NEXT: .cfi_def_cfa wsp, 48 ; CHECK-NEXT: ldp x29, x30, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #48 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret ; ; CHECK-LABEL: OUTLINED_FUNCTION_0: Index: llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/aarch64_generated_funcs.ll.nogenerated.expected =================================================================== --- llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/aarch64_generated_funcs.ll.nogenerated.expected +++ llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/aarch64_generated_funcs.ll.nogenerated.expected @@ -32,8 +32,12 @@ ; CHECK-NEXT: bl OUTLINED_FUNCTION_0 ; CHECK-NEXT: .LBB0_5: ; CHECK-NEXT: mov w0, wzr +; CHECK-NEXT: .cfi_def_cfa wsp, 48 ; CHECK-NEXT: ldp x29, x30, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #48 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %1 = alloca i32, align 4 %2 = alloca i32, align 4 @@ -95,8 +99,12 @@ ; CHECK-NEXT: //NO_APP ; CHECK-NEXT: stp w10, w8, [x29, #-12] ; CHECK-NEXT: stp w9, w11, [sp, #12] +; CHECK-NEXT: .cfi_def_cfa wsp, 48 ; CHECK-NEXT: ldp x29, x30, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #48 +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 ; CHECK-NEXT: ret %1 = alloca i32, align 4 %2 = alloca i32, align 4