Index: llvm/lib/CodeGen/MachineOutliner.cpp =================================================================== --- llvm/lib/CodeGen/MachineOutliner.cpp +++ llvm/lib/CodeGen/MachineOutliner.cpp @@ -1163,6 +1163,14 @@ for (auto I = FirstCand.front(), E = std::next(FirstCand.back()); I != E; ++I) { MachineInstr *NewMI = MF.CloneMachineInstr(&*I); + if (I->isCFIInstruction()) { + MachineFunction *OriginalMF = I->getMF(); + const std::vector &Instrs = + OriginalMF->getFrameInstructions(); + unsigned CFIIndex = NewMI->getOperand(0).getCFIIndex(); + MCCFIInstruction CFI = Instrs[CFIIndex]; + (void)MF.addFrameInst(CFI); + } NewMI->dropMemRefs(MF); // Don't keep debug information for outlined instructions. @@ -1331,7 +1339,6 @@ } LLVM_DEBUG(dbgs() << "OutlinedSomething = " << OutlinedSomething << "\n";); - return OutlinedSomething; } Index: llvm/lib/Target/AArch64/AArch64InstrInfo.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -24,9 +24,9 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" @@ -5862,6 +5862,33 @@ return C.getMF()->getFunction().hasFnAttribute("branch-target-enforcement"); }); + // We check to see if CFI Instructions are present, and if they are + // we add it to a vector. We loop over the instructions in each candidate and + // colllect CFI instructions to compare against the CFI Instructions in the + // original function's frame. We must check this since if we outline one of + // the CFI instructions in a function, we have to outline them all for + // correctness. If we do not, the address offsets will be incorrect between + // the two sections of the program + bool HasCFI = false; + for (outliner::Candidate C : RepeatedSequenceLocs) { + std::vector CFIInstructions = + C.getMF()->getFrameInstructions(); + SmallVector CFIInstructionsPresent; + MachineBasicBlock::iterator Mit = C.front(); + for (unsigned Loc = C.getStartIdx(); Loc < C.getEndIdx() + 1; Loc++) { + if (Mit->isCFIInstruction()) { + HasCFI = true; + unsigned CFIIndex = Mit->getOperand(0).getCFIIndex(); + MCCFIInstruction CFI = CFIInstructions[CFIIndex]; + CFIInstructionsPresent.push_back(CFI); + } + Mit++; + } + + if (HasCFI && CFIInstructionsPresent.size() != CFIInstructions.size()) + return outliner::OutlinedFunction(); + } + // Returns true if an instructions is safe to fix up, false otherwise. auto IsSafeToFixup = [this, &TRI](MachineInstr &MI) { if (MI.isCall()) @@ -6037,6 +6064,11 @@ } } + // If we have CFI instructions, we can only outline if the outlined section + // can be a tail call + if (FrameID != MachineOutlinerTailCall && HasCFI) + return outliner::OutlinedFunction(); + return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, NumBytesToCreateFrame, FrameID); } @@ -6159,12 +6191,13 @@ return outliner::InstrType::Illegal; // We can only outline these if we will tail call the outlined function, or - // fix up the CFI offsets. For the sake of safety, don't outline CFI - // instructions. + // fix up the CFI offsets. Currently, CFI instructions are outlined only if + // in a tail call. // - // FIXME: If the proper fixups are implemented, this should be possible. + // FIXME: If the proper fixups for the offset are implemented, this should be + // possible. if (MI.isCFIInstruction()) - return outliner::InstrType::Illegal; + return outliner::InstrType::Legal; // Don't allow debug values to impact outlining type. if (MI.isDebugInstr() || MI.isIndirectDebugValue()) Index: llvm/lib/Target/X86/X86InstrInfo.cpp =================================================================== --- llvm/lib/Target/X86/X86InstrInfo.cpp +++ llvm/lib/Target/X86/X86InstrInfo.cpp @@ -8682,6 +8682,33 @@ return Sum + 1; }); + // We check to see if CFI Instructions are present, and if they are + // we add it to a vector. We loop over the instructions in each candidate and + // colllect CFI instructions to compare against the CFI Instructions in the + // original function's frame. We must check this since if we outline one of + // the CFI instructions in a function, we have to outline them all for + // correctness. If we do not, the address offsets will be incorrect between + // the two sections of the program + bool HasCFI = false; + for (outliner::Candidate C : RepeatedSequenceLocs) { + std::vector CFIInstructions = + C.getMF()->getFrameInstructions(); + SmallVector CFIInstructionsPresent; + MachineBasicBlock::iterator Mit = C.front(); + for (unsigned Loc = C.getStartIdx(); Loc < C.getEndIdx() + 1; Loc++) { + if (Mit->isCFIInstruction()) { + HasCFI = true; + unsigned CFIIndex = Mit->getOperand(0).getCFIIndex(); + MCCFIInstruction CFI = CFIInstructions[CFIIndex]; + CFIInstructionsPresent.push_back(CFI); + } + Mit++; + } + + if (HasCFI && CFIInstructionsPresent.size() != CFIInstructions.size()) + return outliner::OutlinedFunction(); + } + // FIXME: Use real size in bytes for call and ret instructions. if (RepeatedSequenceLocs[0].back()->isTerminator()) { for (outliner::Candidate &C : RepeatedSequenceLocs) @@ -8693,6 +8720,9 @@ ); } + if (HasCFI) + return outliner::OutlinedFunction(); + for (outliner::Candidate &C : RepeatedSequenceLocs) C.setCallInfo(MachineOutlinerDefault, 1); Index: llvm/test/CodeGen/AArch64/machine-outliner-cfi-tail.mir =================================================================== --- /dev/null +++ llvm/test/CodeGen/AArch64/machine-outliner-cfi-tail.mir @@ -0,0 +1,69 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=aarch64-apple-unknown -run-pass=machine-outliner -verify-machineinstrs %s -o - | FileCheck %s + +# Outlining CFI instructions is unsafe if it is not tail called, but otherwise, +# it requires fixups. Show that we include CFI instructions in tail call +# outlined sequences right now. + +--- | + define void @foo() #0 { ret void } + define void @bar() #0 { ret void } + define void @baz() #0 { ret void } + attributes #0 = { noredzone } +... +--- +name: foo +tracksRegLiveness: true +body: | + bb.0: + liveins: $lr + ; CHECK-LABEL: name: foo + ; CHECK: liveins: $lr + ; CHECK: TCRETURNdi @OUTLINED_FUNCTION_0, 0, implicit $sp, implicit-def $w9, implicit-def $w10, implicit-def $w11, implicit-def $w12, implicit-def $w13, implicit-def $w14, implicit-def $w15, implicit $wzr, implicit $sp + $w9 = ORRWri $wzr, 1 + $w10 = ORRWri $wzr, 2 + $w11 = ORRWri $wzr, 3 + frame-setup CFI_INSTRUCTION def_cfa $w29, 16 + $w12 = ORRWri $wzr, 1 + $w13 = ORRWri $wzr, 2 + $w14 = ORRWri $wzr, 3 + $w15 = ORRWri $wzr, 4 + RET undef $lr +... +--- +name: bar +tracksRegLiveness: true +body: | + bb.0: + liveins: $lr + ; CHECK-LABEL: name: bar + ; CHECK: liveins: $lr + ; CHECK: TCRETURNdi @OUTLINED_FUNCTION_0, 0, implicit $sp, implicit-def $w9, implicit-def $w10, implicit-def $w11, implicit-def $w12, implicit-def $w13, implicit-def $w14, implicit-def $w15, implicit $wzr, implicit $sp + $w9 = ORRWri $wzr, 1 + $w10 = ORRWri $wzr, 2 + $w11 = ORRWri $wzr, 3 + frame-setup CFI_INSTRUCTION def_cfa $w29, 16 + $w12 = ORRWri $wzr, 1 + $w13 = ORRWri $wzr, 2 + $w14 = ORRWri $wzr, 3 + $w15 = ORRWri $wzr, 4 + RET undef $lr +... +--- +name: baz +tracksRegLiveness: true +body: | + bb.0: + liveins: $lr + ; CHECK-LABEL: name: baz + ; CHECK: liveins: $lr + ; CHECK: TCRETURNdi @OUTLINED_FUNCTION_0, 0, implicit $sp, implicit-def $w9, implicit-def $w10, implicit-def $w11, implicit-def $w12, implicit-def $w13, implicit-def $w14, implicit-def $w15, implicit $wzr, implicit $sp + $w9 = ORRWri $wzr, 1 + $w10 = ORRWri $wzr, 2 + $w11 = ORRWri $wzr, 3 + frame-setup CFI_INSTRUCTION def_cfa $w29, 16 + $w12 = ORRWri $wzr, 1 + $w13 = ORRWri $wzr, 2 + $w14 = ORRWri $wzr, 3 + $w15 = ORRWri $wzr, 4 + RET undef $lr Index: llvm/test/CodeGen/AArch64/machine-outliner-cfi.mir =================================================================== --- llvm/test/CodeGen/AArch64/machine-outliner-cfi.mir +++ llvm/test/CodeGen/AArch64/machine-outliner-cfi.mir @@ -1,9 +1,9 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=aarch64-apple-unknown -run-pass=machine-outliner -verify-machineinstrs %s -o - | FileCheck %s -# Outlining CFI instructions is unsafe. It is possible if the call is tail -# called, but otherwise, it requires fixups. Show that we don't include CFI -# instructions in outlined sequences right now. +# Outlining CFI instructions is unsafe if it is not tail called, but otherwise, +# it requires fixups. Show that we don't include CFI instructions in non +# tail call outlined sequences right now. --- | define void @foo() #0 { ret void } @@ -19,9 +19,17 @@ liveins: $lr ; CHECK-LABEL: name: foo ; CHECK: liveins: $lr + ; CHECK: $w9 = ORRWri $wzr, 1 + ; CHECK: $w10 = ORRWri $wzr, 2 + ; CHECK: $w11 = ORRWri $wzr, 3 ; CHECK: frame-setup CFI_INSTRUCTION def_cfa $w29, 16 - ; CHECK: TCRETURNdi @OUTLINED_FUNCTION_0, 0, implicit $sp, implicit-def $w12, implicit-def $w13, implicit-def $w14, implicit-def $w15 + ; CHECK: $w20 = ORRWri $wzr, 1 + ; CHECK: TCRETURNdi @OUTLINED_FUNCTION_0, 0, implicit $sp, implicit-def $w12, implicit-def $w13, implicit-def $w14, implicit-def $w15, implicit $wzr, implicit $sp + $w9 = ORRWri $wzr, 1 + $w10 = ORRWri $wzr, 2 + $w11 = ORRWri $wzr, 3 frame-setup CFI_INSTRUCTION def_cfa $w29, 16 + $w20 = ORRWri $wzr, 1 $w12 = ORRWri $wzr, 1 $w13 = ORRWri $wzr, 2 $w14 = ORRWri $wzr, 3 @@ -36,9 +44,17 @@ liveins: $lr ; CHECK-LABEL: name: bar ; CHECK: liveins: $lr + ; CHECK: $w9 = ORRWri $wzr, 1 + ; CHECK: $w10 = ORRWri $wzr, 2 + ; CHECK: $w11 = ORRWri $wzr, 3 ; CHECK: frame-setup CFI_INSTRUCTION def_cfa $w29, 16 - ; CHECK: TCRETURNdi @OUTLINED_FUNCTION_0, 0, implicit $sp, implicit-def $w12, implicit-def $w13, implicit-def $w14, implicit-def $w15 + ; CHECK: $w21 = ORRWri $wzr, 1 + ; CHECK: TCRETURNdi @OUTLINED_FUNCTION_0, 0, implicit $sp, implicit-def $w12, implicit-def $w13, implicit-def $w14, implicit-def $w15, implicit $wzr, implicit $sp + $w9 = ORRWri $wzr, 1 + $w10 = ORRWri $wzr, 2 + $w11 = ORRWri $wzr, 3 frame-setup CFI_INSTRUCTION def_cfa $w29, 16 + $w21 = ORRWri $wzr, 1 $w12 = ORRWri $wzr, 1 $w13 = ORRWri $wzr, 2 $w14 = ORRWri $wzr, 3 @@ -53,9 +69,17 @@ liveins: $lr ; CHECK-LABEL: name: baz ; CHECK: liveins: $lr + ; CHECK: $w9 = ORRWri $wzr, 1 + ; CHECK: $w10 = ORRWri $wzr, 2 + ; CHECK: $w11 = ORRWri $wzr, 3 ; CHECK: frame-setup CFI_INSTRUCTION def_cfa $w29, 16 - ; CHECK: TCRETURNdi @OUTLINED_FUNCTION_0, 0, implicit $sp, implicit-def $w12, implicit-def $w13, implicit-def $w14, implicit-def $w15 + ; CHECK: $w22 = ORRWri $wzr, 1 + ; CHECK: TCRETURNdi @OUTLINED_FUNCTION_0, 0, implicit $sp, implicit-def $w12, implicit-def $w13, implicit-def $w14, implicit-def $w15, implicit $wzr, implicit $sp + $w9 = ORRWri $wzr, 1 + $w10 = ORRWri $wzr, 2 + $w11 = ORRWri $wzr, 3 frame-setup CFI_INSTRUCTION def_cfa $w29, 16 + $w22 = ORRWri $wzr, 1 $w12 = ORRWri $wzr, 1 $w13 = ORRWri $wzr, 2 $w14 = ORRWri $wzr, 3 Index: llvm/test/CodeGen/AArch64/machine-outliner-remarks.ll =================================================================== --- llvm/test/CodeGen/AArch64/machine-outliner-remarks.ll +++ llvm/test/CodeGen/AArch64/machine-outliner-remarks.ll @@ -4,7 +4,7 @@ ; CHECK-SAME: Bytes from outlining all occurrences (16) >= ; CHECK-SAME: Unoutlined instruction bytes (16) ; CHECK-SAME: (Also found at: ) -; CHECK: remark: :0:0: Saved 36 bytes by outlining 11 instructions +; CHECK: remark: :0:0: Saved 40 bytes by outlining 13 instructions ; CHECK-SAME: from 2 locations. (Found at: , ; CHECK-SAME: ) ; RUN: llc %s -enable-machine-outliner -mtriple=aarch64-unknown-unknown -o /dev/null -pass-remarks-missed=machine-outliner -pass-remarks-output=%t.yaml @@ -38,10 +38,10 @@ ; YAML-NEXT: Function: OUTLINED_FUNCTION_0 ; YAML-NEXT: Args: ; YAML-NEXT: - String: 'Saved ' -; YAML-NEXT: - OutliningBenefit: '36' +; YAML-NEXT: - OutliningBenefit: '40' ; YAML-NEXT: - String: ' bytes by ' ; YAML-NEXT: - String: 'outlining ' -; YAML-NEXT: - Length: '11' +; YAML-NEXT: - Length: '13' ; YAML-NEXT: - String: ' instructions ' ; YAML-NEXT: - String: 'from ' ; YAML-NEXT: - NumOccurrences: '2'