diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp --- a/llvm/lib/CodeGen/TargetInstrInfo.cpp +++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp @@ -1651,10 +1651,25 @@ // Some instrumentations create special TargetOpcode at the start which // expands to special code sequences which must be present. auto First = MBB.getFirstNonDebugInstr(); - if (First != MBB.end() && - (First->getOpcode() == TargetOpcode::FENTRY_CALL || - First->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_ENTER)) + if (First == MBB.end()) + return true; + + if (First->getOpcode() == TargetOpcode::FENTRY_CALL || + First->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_ENTER) + return false; + + // Some instrumentations create special pseudo-instructions at or just before + // the end that must be present. + auto Last = MBB.getLastNonDebugInstr(); + if (Last->getOpcode() == TargetOpcode::PATCHABLE_RET || + Last->getOpcode() == TargetOpcode::PATCHABLE_TAIL_CALL) return false; + if (Last != First && Last->isReturn()) { + --Last; + if (Last->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_EXIT || + Last->getOpcode() == TargetOpcode::PATCHABLE_TAIL_CALL) + return false; + } return true; } diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -126,6 +126,12 @@ if (NumBytes == 0) NumBytes = 4; break; + case TargetOpcode::PATCHABLE_FUNCTION_ENTER: + case TargetOpcode::PATCHABLE_FUNCTION_EXIT: + // An XRay sled can be 4 bytes of alignment plus a 32-byte block. + NumBytes = 36; + break; + case AArch64::SPACE: NumBytes = MI.getOperand(1).getImm(); break; diff --git a/llvm/test/CodeGen/AArch64/branch-relax-xray.ll b/llvm/test/CodeGen/AArch64/branch-relax-xray.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/branch-relax-xray.ll @@ -0,0 +1,42 @@ +; RUN: llc -mtriple=aarch64-unknown-linux-gnu -aarch64-tbz-offset-bits=4 -aarch64-cbz-offset-bits=4 < %s | FileCheck %s + +;; Check that branch relaxation accounts for the size of xray EXIT sleds +;; Note that TAIL_CALL sleds don't exist on AArch64 and don't need a test. +define void @exit(i1 zeroext %0) nounwind "function-instrument"="xray-always" { +; CHECK-LABEL: exit: +; CHECK-NEXT: .Lfunc_begin0: +; CHECK-NEXT: // %bb.0: +; CHECK-NEXT: .p2align 2 +; CHECK-NEXT: .Lxray_sled_0: +; CHECK-NEXT: b #32 +; CHECK-COUNT-7: nop +; CHECK-NOT: nop +; CHECK: tbnz +; CHECK-SAME: [[FALLTHROUGH:.LBB[0-9_]+]] +; CHECK-NEXT: b +; CHECK-SAME: [[OUT_OF_RANGE:.LBB[0-9_]+]] +; CHECK-NEXT: [[FALLTHROUGH]]: +; CHECK-NEXT: bl bar +; CHECK: .p2align 2 +; CHECK-NEXT: .Lxray_sled_1: +; CHECK-NEXT: b #32 +; CHECK-COUNT-7: nop +; CHECK-NOT: nop +; CHECK-NEXT: .Ltmp1: +; CHECK-NEXT: ret +; CHECK-NEXT: [[OUT_OF_RANGE]]: +; CHECK-SAME: // %end2 +; CHECK-NEXT: bl baz + br i1 %0, label %end1, label %end2 + +end1: + %2 = call i32 @bar() + ret void + +end2: + %3 = call i32 @baz() + ret void +} + +declare i32 @bar() +declare i32 @baz() \ No newline at end of file diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-patchable.ll b/llvm/test/CodeGen/AArch64/machine-outliner-patchable.ll --- a/llvm/test/CodeGen/AArch64/machine-outliner-patchable.ll +++ b/llvm/test/CodeGen/AArch64/machine-outliner-patchable.ll @@ -11,7 +11,7 @@ ; CHECK-NEXT: // %bb.0: ; CHECK-NEXT: # FEntry call ; CHECK: // %bb.1: -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: bl [[OUTLINED_FUNCTION:OUTLINED_FUNCTION_[0-9]+]] entry: br i1 %a, label %if.then, label %if.end if.then: @@ -27,7 +27,7 @@ ; CHECK-NEXT: // %bb.0: ; CHECK-NEXT: # FEntry call ; CHECK: // %bb.1: -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: bl [[OUTLINED_FUNCTION]] entry: br i1 %a, label %if.then, label %if.end if.then: @@ -47,7 +47,7 @@ ; CHECK-NEXT: nop ; CHECK-NEXT: nop ; CHECK: // %bb.1: -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: bl [[OUTLINED_FUNCTION]] entry: br i1 %a, label %if.then, label %if.end if.then: @@ -65,7 +65,7 @@ ; CHECK-NEXT: nop ; CHECK-NEXT: nop ; CHECK: // %bb.1: -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: bl [[OUTLINED_FUNCTION]] entry: br i1 %a, label %if.then, label %if.end if.then: @@ -84,7 +84,7 @@ ; CHECK-NEXT: .p2align 2 ; CHECK-NEXT: .Lxray_sled_0: ; CHECK: // %bb.1: -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: bl [[OUTLINED_FUNCTION]] entry: br i1 %a, label %if.then, label %if.end if.then: @@ -102,7 +102,7 @@ ; CHECK-NEXT: .p2align 2 ; CHECK-NEXT: .Lxray_sled_2: ; CHECK: // %bb.1: -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: bl [[OUTLINED_FUNCTION]] entry: br i1 %a, label %if.then, label %if.end if.then: @@ -112,3 +112,12 @@ call void @foo(i32 5, i32 6, i32 7, i32 8) ret void } + +;; Make sure that OUTLINED_FUNCTION contains the right instructions +; CHECK: [[OUTLINED_FUNCTION]]: +; CHECK-NEXT: // %bb.0: +; CHECK-NEXT: mov w0, #1 +; CHECK-NEXT: mov w1, #2 +; CHECK-NEXT: mov w2, #3 +; CHECK-NEXT: mov w3, #4 +; CHECK-NEXT: b foo diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-patchable.mir b/llvm/test/CodeGen/AArch64/machine-outliner-patchable.mir new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/machine-outliner-patchable.mir @@ -0,0 +1,162 @@ +# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass machine-outliner -verify-machineinstrs -enable-machine-outliner %s -o - | FileCheck %s +--- | + ; Function Attrs: minsize + declare void @foo(i32, i32, i32, i32) #0 + + ; Function Attrs: nounwind + define void @xray0(i1 %a) #1 { + entry: + br i1 %a, label %if.then, label %if.end + + if.then: ; preds = %entry + call void @foo(i32 1, i32 2, i32 3, i32 4) + br label %if.end + + if.end: ; preds = %if.then, %entry + call void @foo(i32 5, i32 6, i32 7, i32 8) + ret void + } + + ; Function Attrs: nounwind + define void @xray1(i1 %a) #1 { + entry: + br i1 %a, label %if.then, label %if.end + + if.then: ; preds = %entry + call void @foo(i32 1, i32 2, i32 3, i32 4) + br label %if.end + + if.end: ; preds = %if.then, %entry + call void @foo(i32 5, i32 6, i32 7, i32 8) + ret void + } + + attributes #0 = { minsize } + attributes #1 = { nounwind "function-instrument"="xray-always" } + +... +--- +name: xray0 +tracksRegLiveness: true +liveins: + - { reg: '$w0', virtual-reg: '' } +stack: + - { id: 0, name: '', type: spill-slot, offset: -16, size: 8, alignment: 16, + stack-id: default, callee-saved-register: '$lr', callee-saved-restored: true, + debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } +machineFunctionInfo: + hasRedZone: false +body: | + ; CHECK-LABEL: name: xray0 + ; CHECK: bb.0.entry: + ; CHECK: PATCHABLE_FUNCTION_ENTER + ; CHECK: bb.1.if.then: + ; CHECK: BL @[[OUTLINED_FUNCTION:OUTLINED_FUNCTION_[0-9]]] + ; CHECK: bb.2.if.end: + ; CHECK-NEXT: $w0 = MOVZWi 5, 0 + ; CHECK-NEXT: $w1 = MOVZWi 6, 0 + ; CHECK-NEXT: $w2 = MOVZWi 7, 0 + ; CHECK-NEXT: $w3 = MOVZWi 8, 0 + ; CHECK-NEXT: BL @foo, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit-def $sp + ; CHECK: $w0 = MOVZWi 5, 0 + ; CHECK-NEXT: $w1 = MOVZWi 6, 0 + ; CHECK-NEXT: PATCHABLE_FUNCTION_EXIT + ; CHECK-NEXT: RET undef $lr + + bb.0.entry: + successors: %bb.1(0x40000000), %bb.2(0x40000000) + liveins: $w0, $lr + + PATCHABLE_FUNCTION_ENTER + early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.0) + TBZW killed renamable $w0, 0, %bb.2 + + bb.1.if.then: + successors: %bb.2(0x80000000) + + $w0 = MOVZWi 1, 0 + $w1 = MOVZWi 2, 0 + $w2 = MOVZWi 3, 0 + $w3 = MOVZWi 4, 0 + BL @foo, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit-def $sp + + bb.2.if.end: + $w0 = MOVZWi 5, 0 + $w1 = MOVZWi 6, 0 + $w2 = MOVZWi 7, 0 + $w3 = MOVZWi 8, 0 + BL @foo, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit-def $sp + early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.0) + $w0 = MOVZWi 5, 0 + $w1 = MOVZWi 6, 0 + PATCHABLE_FUNCTION_EXIT + RET undef $lr + +... +--- +name: xray1 +tracksRegLiveness: true +liveins: + - { reg: '$w0', virtual-reg: '' } +stack: + - { id: 0, name: '', type: spill-slot, offset: -16, size: 8, alignment: 16, + stack-id: default, callee-saved-register: '$lr', callee-saved-restored: true, + debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } +machineFunctionInfo: + hasRedZone: false +body: | + ; CHECK-LABEL: name: xray1 + ; CHECK: bb.0.entry: + ; CHECK: PATCHABLE_FUNCTION_ENTER + ; CHECK: bb.1.if.then: + ; CHECK: BL @[[OUTLINED_FUNCTION]] + ; CHECK: bb.2.if.end: + ; CHECK-NEXT: $w0 = MOVZWi 5, 0 + ; CHECK-NEXT: $w1 = MOVZWi 6, 0 + ; CHECK-NEXT: $w2 = MOVZWi 7, 0 + ; CHECK-NEXT: $w3 = MOVZWi 8, 0 + ; CHECK-NEXT: BL @foo, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit-def $sp + ; CHECK: $w0 = MOVZWi 5, 0 + ; CHECK-NEXT: $w1 = MOVZWi 6, 0 + ; CHECK-NEXT: PATCHABLE_FUNCTION_EXIT + ; CHECK-NEXT: RET undef $lr + + bb.0.entry: + successors: %bb.1(0x40000000), %bb.2(0x40000000) + liveins: $w0, $lr + + PATCHABLE_FUNCTION_ENTER + early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.0) + TBZW killed renamable $w0, 0, %bb.2 + + bb.1.if.then: + successors: %bb.2(0x80000000) + + $w0 = MOVZWi 1, 0 + $w1 = MOVZWi 2, 0 + $w2 = MOVZWi 3, 0 + $w3 = MOVZWi 4, 0 + BL @foo, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit-def $sp + + bb.2.if.end: + $w0 = MOVZWi 5, 0 + $w1 = MOVZWi 6, 0 + $w2 = MOVZWi 7, 0 + $w3 = MOVZWi 8, 0 + BL @foo, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit-def $sp + early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.0) + $w0 = MOVZWi 5, 0 + $w1 = MOVZWi 6, 0 + PATCHABLE_FUNCTION_EXIT + RET undef $lr + + ; CHECK: name: [[OUTLINED_FUNCTION]] + ; CHECK: bb.0: + ; CHECK: $w0 = MOVZWi 1, 0 + ; CHECK-NEXT: $w1 = MOVZWi 2, 0 + ; CHECK-NEXT: $w2 = MOVZWi 3, 0 + ; CHECK-NEXT: $w3 = MOVZWi 4, 0 + ; CHECK-NEXT: TCRETURNdi @foo, 0, implicit $sp + +... +