diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp --- a/llvm/lib/CodeGen/TargetInstrInfo.cpp +++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp @@ -1651,10 +1651,25 @@ // Some instrumentations create special TargetOpcode at the start which // expands to special code sequences which must be present. auto First = MBB.getFirstNonDebugInstr(); - if (First != MBB.end() && - (First->getOpcode() == TargetOpcode::FENTRY_CALL || - First->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_ENTER)) + if (First == MBB.end()) + return true; + + if (First->getOpcode() == TargetOpcode::FENTRY_CALL || + First->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_ENTER) + return false; + + // Some instrumentations create special pseudo-instructions at or just before + // the end that must be present. + auto Last = MBB.getLastNonDebugInstr(); + if (Last->getOpcode() == TargetOpcode::PATCHABLE_RET || + Last->getOpcode() == TargetOpcode::PATCHABLE_TAIL_CALL) return false; + if (Last != First && Last->isReturn()) { + --Last; + if (Last->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_EXIT || + Last->getOpcode() == TargetOpcode::PATCHABLE_TAIL_CALL) + return false; + } return true; } diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -126,6 +126,12 @@ if (NumBytes == 0) NumBytes = 4; break; + case TargetOpcode::PATCHABLE_FUNCTION_ENTER: + case TargetOpcode::PATCHABLE_FUNCTION_EXIT: + // An XRay sled can be 4 bytes of alignment plus a 32-byte block. + NumBytes = 36; + break; + case AArch64::SPACE: NumBytes = MI.getOperand(1).getImm(); break; diff --git a/llvm/test/CodeGen/AArch64/branch-relax-xray.ll b/llvm/test/CodeGen/AArch64/branch-relax-xray.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/branch-relax-xray.ll @@ -0,0 +1,42 @@ +; RUN: llc -mtriple=aarch64-unknown-linux-gnu -aarch64-tbz-offset-bits=4 -aarch64-cbz-offset-bits=4 < %s | FileCheck %s + +;; Check that branch relaxation accounts for the size of xray EXIT sleds +;; Note that TAIL_CALL sleds don't exist on AArch64 and don't need a test. +define void @exit(i1 zeroext %0) nounwind "function-instrument"="xray-always" { +; CHECK-LABEL: exit: +; CHECK-NEXT: .Lfunc_begin0: +; CHECK-NEXT: // %bb.0: +; CHECK-NEXT: .p2align 2 +; CHECK-NEXT: .Lxray_sled_0: +; CHECK-NEXT: b #32 +; CHECK-COUNT-7: nop +; CHECK-NOT: nop +; CHECK: tbnz +; CHECK-SAME: [[FALLTHROUGH:.LBB[0-9_]+]] +; CHECK-NEXT: b +; CHECK-SAME: [[OUT_OF_RANGE:.LBB[0-9_]+]] +; CHECK-NEXT: [[FALLTHROUGH]]: +; CHECK-NEXT: bl bar +; CHECK: .p2align 2 +; CHECK-NEXT: .Lxray_sled_1: +; CHECK-NEXT: b #32 +; CHECK-COUNT-7: nop +; CHECK-NOT: nop +; CHECK-NEXT: .Ltmp1: +; CHECK-NEXT: ret +; CHECK-NEXT: [[OUT_OF_RANGE]]: +; CHECK-SAME: // %end2 +; CHECK-NEXT: bl baz + br i1 %0, label %end1, label %end2 + +end1: + %2 = call i32 @bar() + ret void + +end2: + %3 = call i32 @baz() + ret void +} + +declare i32 @bar() +declare i32 @baz() \ No newline at end of file diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-patchable.ll b/llvm/test/CodeGen/AArch64/machine-outliner-patchable.ll --- a/llvm/test/CodeGen/AArch64/machine-outliner-patchable.ll +++ b/llvm/test/CodeGen/AArch64/machine-outliner-patchable.ll @@ -11,7 +11,7 @@ ; CHECK-NEXT: // %bb.0: ; CHECK-NEXT: # FEntry call ; CHECK: // %bb.1: -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: bl [[OUTLINED_FUNCTION:OUTLINED_FUNCTION_[0-9]+]] entry: br i1 %a, label %if.then, label %if.end if.then: @@ -27,7 +27,7 @@ ; CHECK-NEXT: // %bb.0: ; CHECK-NEXT: # FEntry call ; CHECK: // %bb.1: -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: bl [[OUTLINED_FUNCTION]] entry: br i1 %a, label %if.then, label %if.end if.then: @@ -47,7 +47,7 @@ ; CHECK-NEXT: nop ; CHECK-NEXT: nop ; CHECK: // %bb.1: -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: bl [[OUTLINED_FUNCTION]] entry: br i1 %a, label %if.then, label %if.end if.then: @@ -65,7 +65,7 @@ ; CHECK-NEXT: nop ; CHECK-NEXT: nop ; CHECK: // %bb.1: -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: bl [[OUTLINED_FUNCTION]] entry: br i1 %a, label %if.then, label %if.end if.then: @@ -84,7 +84,7 @@ ; CHECK-NEXT: .p2align 2 ; CHECK-NEXT: .Lxray_sled_0: ; CHECK: // %bb.1: -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: bl [[OUTLINED_FUNCTION]] entry: br i1 %a, label %if.then, label %if.end if.then: @@ -102,7 +102,7 @@ ; CHECK-NEXT: .p2align 2 ; CHECK-NEXT: .Lxray_sled_2: ; CHECK: // %bb.1: -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: bl [[OUTLINED_FUNCTION]] entry: br i1 %a, label %if.then, label %if.end if.then: @@ -112,3 +112,12 @@ call void @foo(i32 5, i32 6, i32 7, i32 8) ret void } + +;; Make sure that OUTLINED_FUNCTION contains the right instructions +; CHECK: [[OUTLINED_FUNCTION]]: +; CHECK-NEXT: // %bb.0: +; CHECK-NEXT: mov w0, #1 +; CHECK-NEXT: mov w1, #2 +; CHECK-NEXT: mov w2, #3 +; CHECK-NEXT: mov w3, #4 +; CHECK-NEXT: b foo