Index: llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp +++ llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp @@ -31,6 +31,7 @@ #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/IR/DebugLoc.h" #include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" @@ -1760,6 +1761,26 @@ return E; } +static MachineBasicBlock::iterator +maybeMoveCFI(MachineInstr &MI, MachineBasicBlock::iterator MaybeCFI) { + if (MaybeCFI->getOpcode() != TargetOpcode::CFI_INSTRUCTION || + !(MI.getFlag(MachineInstr::FrameSetup) || + MI.getFlag(MachineInstr::FrameDestroy)) || + getLdStBaseOp(MI).getReg() != AArch64::SP) + return MI.getParent()->end(); + + const MachineFunction &MF = *MI.getParent()->getParent(); + unsigned CFIIndex = MaybeCFI->getOperand(0).getCFIIndex(); + const MCCFIInstruction &CFI = MF.getFrameInstructions()[CFIIndex]; + switch (CFI.getOperation()) { + case MCCFIInstruction::OpDefCfa: + case MCCFIInstruction::OpDefCfaOffset: + return MaybeCFI; + default: + return MI.getParent()->end(); + } +} + MachineBasicBlock::iterator AArch64LoadStoreOpt::mergeUpdateInsn(MachineBasicBlock::iterator I, MachineBasicBlock::iterator Update, @@ -1769,6 +1790,12 @@ "Unexpected base register update instruction to merge!"); MachineBasicBlock::iterator E = I->getParent()->end(); MachineBasicBlock::iterator NextI = next_nodbg(I, E); + + // If updating the SP and the following instruction is CFA offset related CFI + // instruction move it after the merged instruction. + MachineBasicBlock::iterator CFI = + IsPreIdx ? maybeMoveCFI(*Update, next_nodbg(Update, E)) : E; + // Return the instruction following the merged instruction, which is // the instruction following our unmerged load. Unless that's the add/sub // instruction we're merging, in which case it's the one after that. @@ -1806,7 +1833,10 @@ .setMemRefs(I->memoperands()) .setMIFlags(I->mergeFlagsWith(*Update)); } - (void)MIB; + if (CFI != E) { + MachineBasicBlock *MBB = I->getParent(); + MBB->splice(std::next(MIB.getInstr()->getIterator()), MBB, CFI); + } if (IsPreIdx) { ++NumPreFolded; Index: llvm/test/CodeGen/AArch64/arm64-fp128.ll =================================================================== --- llvm/test/CodeGen/AArch64/arm64-fp128.ll +++ llvm/test/CodeGen/AArch64/arm64-fp128.ll @@ -422,8 +422,8 @@ define fp128 @test_neg(fp128 %in) { ; CHECK-LABEL: test_neg: ; CHECK: // %bb.0: -; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: str q0, [sp, #-16]! +; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: ldrb w8, [sp, #15] ; CHECK-NEXT: eor w8, w8, #0x80 ; CHECK-NEXT: strb w8, [sp, #15] Index: llvm/test/CodeGen/AArch64/fcopysign.ll =================================================================== --- llvm/test/CodeGen/AArch64/fcopysign.ll +++ llvm/test/CodeGen/AArch64/fcopysign.ll @@ -13,11 +13,11 @@ define fp128 @copysign0() { ; CHECK-LABEL: copysign0: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: adrp x8, .LCPI0_0 ; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI0_0] ; CHECK-NEXT: adrp x8, val_double ; CHECK-NEXT: str q0, [sp, #-16]! +; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: ldr x8, [x8, :lo12:val_double] ; CHECK-NEXT: ldrb w9, [sp, #15] ; CHECK-NEXT: and x8, x8, #0x8000000000000000 @@ -36,11 +36,11 @@ define fp128@copysign1() { ; CHECK-LABEL: copysign1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: adrp x8, val_fp128 ; CHECK-NEXT: ldr q0, [x8, :lo12:val_fp128] ; CHECK-NEXT: adrp x8, val_float ; CHECK-NEXT: str q0, [sp, #-16]! +; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: ldr w8, [x8, :lo12:val_float] ; CHECK-NEXT: ldrb w9, [sp, #15] ; CHECK-NEXT: and w8, w8, #0x80000000 Index: llvm/test/CodeGen/AArch64/swifttail-call.ll =================================================================== --- llvm/test/CodeGen/AArch64/swifttail-call.ll +++ llvm/test/CodeGen/AArch64/swifttail-call.ll @@ -36,6 +36,7 @@ ret void ; COMMON: str {{x[0-9]+}}, [sp, #-16]! +; COMMON-NEXT: .cfi_def_cfa_offset 16 ; COMMON-NEXT: b callee_stack8 } Index: llvm/test/CodeGen/AArch64/tail-call.ll =================================================================== --- llvm/test/CodeGen/AArch64/tail-call.ll +++ llvm/test/CodeGen/AArch64/tail-call.ll @@ -36,6 +36,7 @@ ret void ; COMMON: str {{x[0-9]+}}, [sp, #-16]! +; COMMON-NEXT: .cfi_def_cfa_offset 16 ; COMMON-NEXT: b callee_stack8 }