diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -269,14 +269,10 @@ static int64_t getArgumentStackToRestore(MachineFunction &MF, MachineBasicBlock &MBB) { MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr(); - bool IsTailCallReturn = false; - if (MBB.end() != MBBI) { - unsigned RetOpcode = MBBI->getOpcode(); - IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi || - RetOpcode == AArch64::TCRETURNri || - RetOpcode == AArch64::TCRETURNriBTI; - } AArch64FunctionInfo *AFI = MF.getInfo(); + bool IsTailCallReturn = (MBB.end() != MBBI) + ? AArch64InstrInfo::isTailCallReturnInst(*MBBI) + : false; int64_t ArgumentPopSize = 0; if (IsTailCallReturn) { @@ -300,7 +296,6 @@ static bool produceCompactUnwindFrame(MachineFunction &MF); static bool needsWinCFI(const MachineFunction &MF); static StackOffset getSVEStackSize(const MachineFunction &MF); -static bool needsShadowCallStackPrologueEpilogue(MachineFunction &MF); /// Returns true if a homogeneous prolog or epilog code can be emitted /// for the size optimization. If possible, a frame helper call is injected. @@ -613,7 +608,7 @@ } // Shadow call stack uses X18, reset it. - if (needsShadowCallStackPrologueEpilogue(MF)) + if (MFI.needsShadowCallStackPrologueEpilogue(MF)) insertCFISameValue(CFIDesc, MF, MBB, InsertPt, TRI.getDwarfRegNum(AArch64::X18, true)); @@ -1286,19 +1281,6 @@ } } -static bool needsShadowCallStackPrologueEpilogue(MachineFunction &MF) { - if (!(llvm::any_of( - MF.getFrameInfo().getCalleeSavedInfo(), - [](const auto &Info) { return Info.getReg() == AArch64::LR; }) && - MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack))) - return false; - - if (!MF.getSubtarget().isXRegisterReserved(18)) - report_fatal_error("Must reserve x18 to use shadow call stack"); - - return true; -} - static void emitShadowCallStackPrologue(const TargetInstrInfo &TII, MachineFunction &MF, MachineBasicBlock &MBB, @@ -1410,7 +1392,7 @@ DebugLoc DL; const auto &MFnI = *MF.getInfo(); - if (needsShadowCallStackPrologueEpilogue(MF)) + if (MFnI.needsShadowCallStackPrologueEpilogue(MF)) emitShadowCallStackPrologue(*TII, MF, MBB, MBBI, DL, NeedsWinCFI, MFnI.needsDwarfUnwindInfo(MF)); @@ -1917,7 +1899,7 @@ if (NeedsWinCFI) HasWinCFI = true; // AArch64PointerAuth pass will insert SEH_PACSignLR } - if (needsShadowCallStackPrologueEpilogue(MF)) + if (AFI->needsShadowCallStackPrologueEpilogue(MF)) emitShadowCallStackEpilogue(*TII, MF, MBB, MBB.getFirstTerminator(), DL); if (EmitCFI) emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator()); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h @@ -126,6 +126,9 @@ /// Return true if pairing the given load or store may be paired with another. static bool isPairableLdStInst(const MachineInstr &MI); + /// Returns true if MI is one of the TCRETURN* instructions. + static bool isTailCallReturnInst(const MachineInstr &MI); + /// Return the opcode that set flags when possible. The caller is /// responsible for ensuring the opc has a flag setting equivalent. static unsigned convertToFlagSettingOpc(unsigned Opc); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -14,6 +14,7 @@ #include "AArch64InstrInfo.h" #include "AArch64FrameLowering.h" #include "AArch64MachineFunctionInfo.h" +#include "AArch64PointerAuth.h" #include "AArch64Subtarget.h" #include "MCTargetDesc/AArch64AddressingModes.h" #include "Utils/AArch64BaseInfo.h" @@ -2485,6 +2486,20 @@ } } +bool AArch64InstrInfo::isTailCallReturnInst(const MachineInstr &MI) { + switch (MI.getOpcode()) { + default: + assert((!MI.isCall() || !MI.isReturn()) && + "Unexpected instruction - was a new tail call opcode introduced?"); + return false; + case AArch64::TCRETURNdi: + case AArch64::TCRETURNri: + case AArch64::TCRETURNriBTI: + case AArch64::TCRETURNriALL: + return true; + } +} + unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc) { switch (Opc) { default: @@ -8157,12 +8172,24 @@ // necessary. However, at this point we don't know if the outlined function // will have a RET instruction so we assume the worst. const TargetRegisterInfo &TRI = getRegisterInfo(); + // Performing a tail call may require extra checks when PAuth is enabled. + // If PAuth is disabled, set it to zero for uniformity. + unsigned NumBytesToCheckLRInTCEpilogue = 0; if (FirstCand.getMF() ->getInfo() ->shouldSignReturnAddress(true)) { // One PAC and one AUT instructions NumBytesToCreateFrame += 8; + // PAuth is enabled - set extra tail call cost, if any. + auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod(); + NumBytesToCheckLRInTCEpilogue = + AArch64PAuth::getCheckerSizeInBytes(LRCheckMethod); + // Checking the authenticated LR value may significantly impact + // SequenceSize, so account for it for more precise results. + if (isTailCallReturnInst(*RepeatedSequenceLocs[0].back())) + SequenceSize += NumBytesToCheckLRInTCEpilogue; + // We have to check if sp modifying instructions would get outlined. // If so we only allow outlining if sp is unchanged overall, so matching // sub and add instructions are okay to outline, all other sp modifications @@ -8333,7 +8360,8 @@ if (RepeatedSequenceLocs[0].back()->isTerminator()) { FrameID = MachineOutlinerTailCall; NumBytesToCreateFrame = 0; - SetCandidateCallInfo(MachineOutlinerTailCall, 4); + unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue; + SetCandidateCallInfo(MachineOutlinerTailCall, NumBytesForCall); } else if (LastInstrOpcode == AArch64::BL || @@ -8342,7 +8370,7 @@ !HasBTI)) { // FIXME: Do we need to check if the code after this uses the value of LR? FrameID = MachineOutlinerThunk; - NumBytesToCreateFrame = 0; + NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue; SetCandidateCallInfo(MachineOutlinerThunk, 4); } diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h --- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h +++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h @@ -431,6 +431,8 @@ bool shouldSignReturnAddress(const MachineFunction &MF) const; bool shouldSignReturnAddress(bool SpillsLR) const; + bool needsShadowCallStackPrologueEpilogue(MachineFunction &MF) const; + bool shouldSignWithBKey() const { return SignWithBKey; } bool isMTETagged() const { return IsMTETagged; } diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp --- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp @@ -122,11 +122,27 @@ return SpillsLR; } +static bool isLRSpilled(const MachineFunction &MF) { + return llvm::any_of( + MF.getFrameInfo().getCalleeSavedInfo(), + [](const auto &Info) { return Info.getReg() == AArch64::LR; }); +} + bool AArch64FunctionInfo::shouldSignReturnAddress( const MachineFunction &MF) const { - return shouldSignReturnAddress(llvm::any_of( - MF.getFrameInfo().getCalleeSavedInfo(), - [](const auto &Info) { return Info.getReg() == AArch64::LR; })); + return shouldSignReturnAddress(isLRSpilled(MF)); +} + +bool AArch64FunctionInfo::needsShadowCallStackPrologueEpilogue( + MachineFunction &MF) const { + if (!(isLRSpilled(MF) && + MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack))) + return false; + + if (!MF.getSubtarget().isXRegisterReserved(18)) + report_fatal_error("Must reserve x18 to use shadow call stack"); + + return true; } bool AArch64FunctionInfo::needsDwarfUnwindInfo( diff --git a/llvm/lib/Target/AArch64/AArch64PointerAuth.h b/llvm/lib/Target/AArch64/AArch64PointerAuth.h new file mode 100644 --- /dev/null +++ b/llvm/lib/Target/AArch64/AArch64PointerAuth.h @@ -0,0 +1,116 @@ +//===-- AArch64PointerAuth.h -- Harden code using PAuth ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64POINTERAUTH_H +#define LLVM_LIB_TARGET_AARCH64_AARCH64POINTERAUTH_H + +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/Register.h" + +namespace llvm { +namespace AArch64PAuth { + +/// Variants of check performed on an authenticated pointer. +/// +/// In cases such as authenticating the LR value when performing a tail call +/// or when re-signing a signed pointer with a different signing schema, +/// a failed authentication may not generate an exception on its own and may +/// create an authentication or signing oracle if not checked explicitly. +/// +/// A number of check methods modify control flow in a similar way by +/// rewriting the code +/// +/// ``` +/// +/// +/// ``` +/// +/// as follows: +/// +/// ``` +/// +/// +/// ret_block: +/// +/// ... +/// +/// break_block: +/// brk +/// ``` +enum class AuthCheckMethod { + /// Do not check the value at all + None, + /// Perform a load to a temporary register + DummyLoad, + /// Check by comparing bits 62 and 61 of the authenticated address. + /// + /// This method modifies control flow and inserts the following checker: + /// + /// ``` + /// eor Xtmp, Xn, Xn, lsl #1 + /// tbnz Xtmp, #62, break_block + /// ``` + HighBitsNoTBI, + /// Check by comparing the authenticated value with an XPAC-ed one without + /// using PAuth instructions not encoded as HINT. Can only be applied to LR. + /// + /// This method modifies control flow and inserts the following checker: + /// + /// ``` + /// mov Xtmp, LR + /// xpaclri ; encoded as "hint #7" + /// ; Note: at this point, the LR register contains the address as if + /// ; the authentication succeeded and the temporary register contains the + /// ; *real* result of authentication. + /// cmp Xtmp, LR + /// b.ne break_block + /// ``` + XPACHint, +}; + +#define AUTH_CHECK_METHOD_CL_VALUES_COMMON \ + clEnumValN(AArch64PAuth::AuthCheckMethod::None, "none", \ + "Do not check authenticated address"), \ + clEnumValN(AArch64PAuth::AuthCheckMethod::DummyLoad, "load", \ + "Perform dummy load from authenticated address"), \ + clEnumValN(AArch64PAuth::AuthCheckMethod::HighBitsNoTBI, \ + "high-bits-notbi", \ + "Compare bits 62 and 61 of address (TBI should be disabled)") + +#define AUTH_CHECK_METHOD_CL_VALUES_LR \ + AUTH_CHECK_METHOD_CL_VALUES_COMMON, \ + clEnumValN(AArch64PAuth::AuthCheckMethod::XPACHint, "xpac-hint", \ + "Compare with the result of XPACLRI") + +/// Explicitly checks that pointer authentication succeeded. +/// +/// Assuming AuthenticatedReg contains a value returned by one of the AUT* +/// instructions, check the value using Method just before the instruction +/// pointed to by MBBI. If the check succeeds, execution proceeds to the +/// instruction pointed to by MBBI, otherwise a CPU exception is generated. +/// +/// Some of the methods may need to know if the pointer was authenticated +/// using an I-key or D-key and which register can be used as temporary. +/// If an explicit BRK instruction is used to generate an exception, BrkImm +/// specifies its immediate operand. +/// +/// \returns The machine basic block containing the code that is executed +/// after the check succeeds. +MachineBasicBlock &checkAuthenticatedRegister(MachineBasicBlock::iterator MBBI, + AuthCheckMethod Method, + Register AuthenticatedReg, + Register TmpReg, bool UseIKey, + unsigned BrkImm); + +/// Returns the number of bytes added by checkAuthenticatedRegister. +unsigned getCheckerSizeInBytes(AuthCheckMethod Method); + +} // end namespace AArch64PAuth +} // end namespace llvm + +#endif diff --git a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp --- a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp +++ b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp @@ -6,7 +6,10 @@ // //===----------------------------------------------------------------------===// +#include "AArch64PointerAuth.h" + #include "AArch64.h" +#include "AArch64InstrInfo.h" #include "AArch64MachineFunctionInfo.h" #include "AArch64Subtarget.h" #include "llvm/CodeGen/MachineBasicBlock.h" @@ -14,6 +17,7 @@ #include "llvm/CodeGen/MachineModuleInfo.h" using namespace llvm; +using namespace llvm::AArch64PAuth; #define AARCH64_POINTER_AUTH_NAME "AArch64 Pointer Authentication" @@ -30,13 +34,19 @@ StringRef getPassName() const override { return AARCH64_POINTER_AUTH_NAME; } private: + /// An immediate operand passed to BRK instruction, if it is ever emitted. + const unsigned BrkOperand = 0xc471; + const AArch64Subtarget *Subtarget = nullptr; const AArch64InstrInfo *TII = nullptr; + const AArch64RegisterInfo *TRI = nullptr; void signLR(MachineFunction &MF, MachineBasicBlock::iterator MBBI) const; void authenticateLR(MachineFunction &MF, MachineBasicBlock::iterator MBBI) const; + + bool checkAuthenticatedLR(MachineBasicBlock::iterator TI) const; }; } // end anonymous namespace @@ -132,22 +142,191 @@ } } +namespace { + +/// AddressCheckPseudoSourceValue represents the memory access inserted by +/// the AuthCheckMethod::DummyLoad method to trigger a run-time error if the +/// authentication of a pointer failed. +class AddressCheckPseudoSourceValue : public PseudoSourceValue { +public: + AddressCheckPseudoSourceValue(const TargetMachine &TM) + : PseudoSourceValue(TargetCustom, TM) {} + + bool isConstant(const MachineFrameInfo *) const override { return false; } + bool isAliased(const MachineFrameInfo *) const override { return true; } + bool mayAlias(const MachineFrameInfo *) const override { return true; } + void printCustom(raw_ostream &OS) const override { OS << "AddressCheck"; } +}; + +// Mark dummy LDR instruction as volatile to prevent removing it as dead code. +MachineMemOperand *createCheckMemOperand(MachineFunction &MF) { + static AddressCheckPseudoSourceValue CheckPSV(MF.getTarget()); + MachinePointerInfo PointerInfo(&CheckPSV); + auto MOVolatileLoad = + MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile; + + return MF.getMachineMemOperand(PointerInfo, MOVolatileLoad, 4, Align(4)); +} + +} // namespace + +MachineBasicBlock &llvm::AArch64PAuth::checkAuthenticatedRegister( + MachineBasicBlock::iterator MBBI, AuthCheckMethod Method, + Register AuthenticatedReg, Register TmpReg, bool UseIKey, unsigned BrkImm) { + + MachineBasicBlock &MBB = *MBBI->getParent(); + MachineFunction &MF = *MBB.getParent(); + const AArch64Subtarget &Subtarget = MF.getSubtarget(); + const AArch64InstrInfo *TII = Subtarget.getInstrInfo(); + DebugLoc DL = MBBI->getDebugLoc(); + + // First, handle the methods not requiring creating extra MBBs. + switch (Method) { + default: + break; + case AuthCheckMethod::None: + return MBB; + case AuthCheckMethod::DummyLoad: + BuildMI(MBB, MBBI, DL, TII->get(AArch64::LDRWui), getWRegFromXReg(TmpReg)) + .addReg(AArch64::LR) + .addImm(0) + .addMemOperand(createCheckMemOperand(MF)); + return MBB; + } + + // Control flow has to be changed, so arrange new MBBs. + + // At now, at least an AUT* instruction is expected before MBBI + assert(MBBI != MBB.begin() && + "Cannot insert the check at the very beginning of MBB"); + // The block to insert check into. + MachineBasicBlock *CheckBlock = &MBB; + // The remaining part of the original MBB that is executed on success. + MachineBasicBlock *SuccessBlock = MBB.splitAt(*std::prev(MBBI)); + + // The block that explicitly generates a break-point exception on failure. + MachineBasicBlock *BreakBlock = + MF.CreateMachineBasicBlock(MBB.getBasicBlock()); + MF.push_back(BreakBlock); + MBB.splitSuccessor(SuccessBlock, BreakBlock); + + assert(CheckBlock->getFallThrough() == SuccessBlock); + BuildMI(BreakBlock, DL, TII->get(AArch64::BRK)).addImm(BrkImm); + + switch (Method) { + case AuthCheckMethod::None: + case AuthCheckMethod::DummyLoad: + llvm_unreachable("Should be handled above"); + case AuthCheckMethod::HighBitsNoTBI: + BuildMI(CheckBlock, DL, TII->get(AArch64::EORXrs), TmpReg) + .addReg(AuthenticatedReg) + .addReg(AuthenticatedReg) + .addImm(1); + BuildMI(CheckBlock, DL, TII->get(AArch64::TBNZX)) + .addReg(TmpReg) + .addImm(62) + .addMBB(BreakBlock); + return *SuccessBlock; + case AuthCheckMethod::XPACHint: + assert(AuthenticatedReg == AArch64::LR && + "XPACHint mode is only compatible with checking the LR register"); + assert(UseIKey && "XPACHint mode is only compatible with I-keys"); + BuildMI(CheckBlock, DL, TII->get(AArch64::ORRXrs), TmpReg) + .addReg(AArch64::XZR) + .addReg(AArch64::LR) + .addImm(0); + BuildMI(CheckBlock, DL, TII->get(AArch64::XPACLRI)); + BuildMI(CheckBlock, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR) + .addReg(TmpReg) + .addReg(AArch64::LR) + .addImm(0); + BuildMI(CheckBlock, DL, TII->get(AArch64::Bcc)) + .addImm(AArch64CC::NE) + .addMBB(BreakBlock); + return *SuccessBlock; + } +} + +unsigned llvm::AArch64PAuth::getCheckerSizeInBytes(AuthCheckMethod Method) { + switch (Method) { + case AuthCheckMethod::None: + return 0; + case AuthCheckMethod::DummyLoad: + return 4; + case AuthCheckMethod::HighBitsNoTBI: + return 12; + case AuthCheckMethod::XPACHint: + return 20; + } +} + +bool AArch64PointerAuth::checkAuthenticatedLR( + MachineBasicBlock::iterator TI) const { + AuthCheckMethod Method = Subtarget->getAuthenticatedLRCheckMethod(); + + if (Method == AuthCheckMethod::None) + return false; + + assert(!TI->getMF()->hasWinCFI() && "WinCFI is not yet supported"); + + // The following code may create a signing oracle: + // + // + // TCRETURN ; the callee may sign and spill the LR in its prologue + // + // To avoid generating a signing oracle, check the authenticated value + // before possibly re-signing it in the callee, as follows: + // + // + // + // b. break_block + // ret_block: + // TCRETURN + // break_block: + // brk + // + // or just + // + // + // ldr tmp, [lr] + // TCRETURN + + // TmpReg is chosen assuming X16 and X17 are dead after TI. + assert(AArch64InstrInfo::isTailCallReturnInst(*TI) && + "Tail call is expected"); + Register TmpReg = + TI->readsRegister(AArch64::X16, TRI) ? AArch64::X17 : AArch64::X16; + assert(!TI->readsRegister(TmpReg, TRI) && + "More than a single register is used by TCRETURN"); + + checkAuthenticatedRegister(TI, Method, AArch64::LR, TmpReg, /*UseIKey=*/true, + BrkOperand); + + return true; +} + bool AArch64PointerAuth::runOnMachineFunction(MachineFunction &MF) { - if (!MF.getInfo()->shouldSignReturnAddress(true)) + const auto *MFnI = MF.getInfo(); + if (!MFnI->shouldSignReturnAddress(true)) return false; Subtarget = &MF.getSubtarget(); TII = Subtarget->getInstrInfo(); + TRI = Subtarget->getRegisterInfo(); SmallVector DeletedInstrs; + SmallVector TailCallInstrs; + bool Modified = false; + bool HasAuthenticationInstrs = false; for (auto &MBB : MF) { for (auto &MI : MBB) { auto It = MI.getIterator(); switch (MI.getOpcode()) { default: - // do nothing + if (AArch64InstrInfo::isTailCallReturnInst(MI)) + TailCallInstrs.push_back(It); break; case AArch64::PAUTH_PROLOGUE: signLR(MF, It); @@ -158,11 +337,20 @@ authenticateLR(MF, It); DeletedInstrs.push_back(It); Modified = true; + HasAuthenticationInstrs = true; break; } } } + // FIXME Do we need to emit any PAuth-related epilogue code at all + // when SCS is enabled? + if (HasAuthenticationInstrs && + !MFnI->needsShadowCallStackPrologueEpilogue(MF)) { + for (auto TailCall : TailCallInstrs) + Modified |= checkAuthenticatedLR(TailCall); + } + for (auto MI : DeletedInstrs) MI->eraseFromParent(); diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h --- a/llvm/lib/Target/AArch64/AArch64Subtarget.h +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h @@ -16,6 +16,7 @@ #include "AArch64FrameLowering.h" #include "AArch64ISelLowering.h" #include "AArch64InstrInfo.h" +#include "AArch64PointerAuth.h" #include "AArch64RegisterInfo.h" #include "AArch64SelectionDAGInfo.h" #include "llvm/CodeGen/GlobalISel/CallLowering.h" @@ -432,6 +433,9 @@ return "__security_check_cookie_arm64ec"; return "__security_check_cookie"; } + + /// Choose a method of checking LR before performing a tail call. + AArch64PAuth::AuthCheckMethod getAuthenticatedLRCheckMethod() const; }; } // End llvm namespace diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp --- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp @@ -70,6 +70,13 @@ "Force the use of streaming-compatible SVE code for all functions"), cl::Hidden); +static cl::opt + AuthenticatedLRCheckMethod("aarch64-authenticated-lr-check-method", + cl::Hidden, + cl::desc("Override the variant of check applied " + "to authenticated LR during tail call"), + cl::values(AUTH_CHECK_METHOD_CL_VALUES_LR)); + unsigned AArch64Subtarget::getVectorInsertExtractBaseCost() const { if (OverrideVectorInsertExtractBaseCost.getNumOccurrences() > 0) return OverrideVectorInsertExtractBaseCost; @@ -490,3 +497,26 @@ // as we don't yet support the feature in LLVM. return hasSVE() && !isStreaming() && !isStreamingCompatible(); } + +// If return address signing is enabled, tail calls are emitted as follows: +// +// ``` +// +// +// TCRETURN ; the callee may sign and spill the LR in its prologue +// ``` +// +// LR may require explicit checking because if FEAT_FPAC is not implemented +// and LR was tampered with, then `` will not generate an +// exception on its own. Later, if the callee spills the signed LR value and +// neither FEAT_PAuth2 nor FEAT_EPAC are implemented, the valid PAC replaces +// the higher bits of LR thus hiding the authentication failure. +AArch64PAuth::AuthCheckMethod +AArch64Subtarget::getAuthenticatedLRCheckMethod() const { + if (AuthenticatedLRCheckMethod.getNumOccurrences()) + return AuthenticatedLRCheckMethod; + + // At now, use None by default because checks may introduce an unexpected + // performance regression or incompatibility with execute-only mappings. + return AArch64PAuth::AuthCheckMethod::None; +} diff --git a/llvm/test/CodeGen/AArch64/sign-return-address-tailcall.ll b/llvm/test/CodeGen/AArch64/sign-return-address-tailcall.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sign-return-address-tailcall.ll @@ -0,0 +1,121 @@ +; RUN: llc -mtriple=aarch64 -asm-verbose=0 < %s | FileCheck -DAUTIASP="hint #29" --check-prefixes=COMMON %s +; RUN: llc -mtriple=aarch64 -asm-verbose=0 -aarch64-authenticated-lr-check-method=load < %s | FileCheck -DAUTIASP="hint #29" --check-prefixes=COMMON,LDR %s +; RUN: llc -mtriple=aarch64 -asm-verbose=0 -aarch64-authenticated-lr-check-method=high-bits-notbi < %s | FileCheck -DAUTIASP="hint #29" --check-prefixes=COMMON,BITS-NOTBI,BRK %s +; RUN: llc -mtriple=aarch64 -asm-verbose=0 -aarch64-authenticated-lr-check-method=xpac-hint < %s | FileCheck -DAUTIASP="hint #29" -DXPACLRI="hint #7" --check-prefixes=COMMON,XPAC,BRK %s +; RUN: llc -mtriple=aarch64 -asm-verbose=0 -aarch64-authenticated-lr-check-method=xpac-hint -mattr=v8.3a < %s | FileCheck -DAUTIASP="autiasp" -DXPACLRI="xpaclri" --check-prefixes=COMMON,XPAC,BRK %s + +define i32 @tailcall_direct() "sign-return-address"="non-leaf" { +; COMMON-LABEL: tailcall_direct: +; COMMON: str x30, [sp, #-16]! +; COMMON: ldr x30, [sp], #16 +; +; COMMON-NEXT: [[AUTIASP]] +; +; LDR-NEXT: ldr w16, [x30] +; +; BITS-NOTBI-NEXT: eor x16, x30, x30, lsl #1 +; BITS-NOTBI-NEXT: tbnz x16, #62, .[[FAIL:LBB[_0-9]+]] +; +; XPAC-NEXT: mov x16, x30 +; XPAC-NEXT: [[XPACLRI]] +; XPAC-NEXT: cmp x16, x30 +; XPAC-NEXT: b.ne .[[FAIL:LBB[_0-9]+]] +; +; COMMON-NEXT: b callee +; BRK-NEXT: .[[FAIL]]: +; BRK-NEXT: brk #0xc471 + tail call void asm sideeffect "", "~{lr}"() + %call = tail call i32 @callee() + ret i32 %call +} + +define i32 @tailcall_indirect(ptr %fptr) "sign-return-address"="non-leaf" { +; COMMON-LABEL: tailcall_indirect: +; COMMON: str x30, [sp, #-16]! +; COMMON: ldr x30, [sp], #16 +; +; COMMON-NEXT: [[AUTIASP]] +; +; LDR-NEXT: ldr w16, [x30] +; +; BITS-NOTBI-NEXT: eor x16, x30, x30, lsl #1 +; BITS-NOTBI-NEXT: tbnz x16, #62, .[[FAIL:LBB[_0-9]+]] +; +; XPAC-NEXT: mov x16, x30 +; XPAC-NEXT: [[XPACLRI]] +; XPAC-NEXT: cmp x16, x30 +; XPAC-NEXT: b.ne .[[FAIL:LBB[_0-9]+]] +; +; COMMON-NEXT: br x0 +; BRK-NEXT: .[[FAIL]]: +; BRK-NEXT: brk #0xc471 + tail call void asm sideeffect "", "~{lr}"() + %call = tail call i32 %fptr() + ret i32 %call +} + +define i32 @tailcall_direct_noframe() "sign-return-address"="non-leaf" { +; COMMON-LABEL: tailcall_direct_noframe: +; COMMON-NEXT: .cfi_startproc +; COMMON-NEXT: b callee + %call = tail call i32 @callee() + ret i32 %call +} + +define i32 @tailcall_indirect_noframe(ptr %fptr) "sign-return-address"="non-leaf" { +; COMMON-LABEL: tailcall_indirect_noframe: +; COMMON-NEXT: .cfi_startproc +; COMMON-NEXT: br x0 + %call = tail call i32 %fptr() + ret i32 %call +} + +define i32 @tailcall_direct_noframe_sign_all() "sign-return-address"="all" { +; COMMON-LABEL: tailcall_direct_noframe_sign_all: +; COMMON-NOT: str{{.*}}x30 +; COMMON-NOT: ldr{{.*}}x30 +; +; COMMON: [[AUTIASP]] +; +; LDR-NEXT: ldr w16, [x30] +; +; BITS-NOTBI-NEXT: eor x16, x30, x30, lsl #1 +; BITS-NOTBI-NEXT: tbnz x16, #62, .[[FAIL:LBB[_0-9]+]] +; +; XPAC-NEXT: mov x16, x30 +; XPAC-NEXT: [[XPACLRI]] +; XPAC-NEXT: cmp x16, x30 +; XPAC-NEXT: b.ne .[[FAIL:LBB[_0-9]+]] +; +; COMMON-NEXT: b callee +; BRK-NEXT: .[[FAIL]]: +; BRK-NEXT: brk #0xc471 + %call = tail call i32 @callee() + ret i32 %call +} + +define i32 @tailcall_indirect_noframe_sign_all(ptr %fptr) "sign-return-address"="all" { +; COMMON-LABEL: tailcall_indirect_noframe_sign_all: +; COMMON-NOT: str{{.*}}x30 +; COMMON-NOT: ldr{{.*}}x30 +; +; COMMON: [[AUTIASP]] +; +; LDR-NEXT: ldr w16, [x30] +; +; BITS-NOTBI-NEXT: eor x16, x30, x30, lsl #1 +; BITS-NOTBI-NEXT: tbnz x16, #62, .[[FAIL:LBB[_0-9]+]] +; +; XPAC-NEXT: mov x16, x30 +; XPAC-NEXT: [[XPACLRI]] +; XPAC-NEXT: cmp x16, x30 +; XPAC-NEXT: b.ne .[[FAIL:LBB[_0-9]+]] +; +; COMMON-NEXT: br x0 +; BRK-NEXT: .[[FAIL]]: +; BRK-NEXT: brk #0xc471 + %call = tail call i32 %fptr() + ret i32 %call +} + +declare i32 @callee()