Index: llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -1115,6 +1115,14 @@ EmitToStreamer(*OutStreamer, TmpInst); return; } + case AArch64::BLRCall: + case AArch64::BLRCallX16X17: { + MCInst TmpInst; + TmpInst.setOpcode(AArch64::BLR); + TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); + EmitToStreamer(*OutStreamer, TmpInst); + return; + } case AArch64::SpeculationBarrierISBDSBEndBB: { // Print DSB SYS + ISB MCInst TmpInstDSB; Index: llvm/lib/Target/AArch64/AArch64FastISel.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64FastISel.cpp +++ llvm/lib/Target/AArch64/AArch64FastISel.cpp @@ -3270,7 +3270,8 @@ // Issue the call. MachineInstrBuilder MIB; if (Subtarget->useSmallAddressing()) { - const MCInstrDesc &II = TII.get(Addr.getReg() ? AArch64::BLR : AArch64::BL); + const MCInstrDesc &II = + TII.get(Addr.getReg() ? AArch64::BLRCall : AArch64::BL); MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II); if (Symbol) MIB.addSym(Symbol, 0); @@ -3303,7 +3304,7 @@ if (!CallReg) return false; - const MCInstrDesc &II = TII.get(AArch64::BLR); + const MCInstrDesc &II = TII.get(AArch64::BLRCall); CallReg = constrainOperandRegClass(II, CallReg, 0); MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).addReg(CallReg); } Index: llvm/lib/Target/AArch64/AArch64FrameLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64FrameLowering.cpp +++ llvm/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -1126,7 +1126,7 @@ .setMIFlag(MachineInstr::FrameSetup); } - BuildMI(MBB, MBBI, DL, TII->get(AArch64::BLR)) + BuildMI(MBB, MBBI, DL, TII->get(AArch64::BLRCallX16X17)) .addReg(AArch64::X16, RegState::Kill) .addReg(AArch64::X15, RegState::Implicit | RegState::Define) .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead) Index: llvm/lib/Target/AArch64/AArch64InstrInfo.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -6092,7 +6092,7 @@ } else if (LastInstrOpcode == AArch64::BL || - (LastInstrOpcode == AArch64::BLR && !HasBTI)) { + (LastInstrOpcode == AArch64::BLRCall && !HasBTI)) { // FIXME: Do we need to check if the code after this uses the value of LR? FrameID = MachineOutlinerThunk; NumBytesToCreateFrame = 0; @@ -6409,7 +6409,9 @@ // as a tail-call. Whitelist the call instructions we know about so we // don't get unexpected results with call pseudo-instructions. auto UnknownCallOutlineType = outliner::InstrType::Illegal; - if (MI.getOpcode() == AArch64::BLR || MI.getOpcode() == AArch64::BL) + if (MI.getOpcode() == AArch64::BLRCall || + MI.getOpcode() == AArch64::BLRCallX16X17 || + MI.getOpcode() == AArch64::BL) UnknownCallOutlineType = outliner::InstrType::LegalTerminator; if (!Callee) @@ -6557,7 +6559,7 @@ if (Call->getOpcode() == AArch64::BL) { TailOpcode = AArch64::TCRETURNdi; } else { - assert(Call->getOpcode() == AArch64::BLR); + assert(Call->getOpcode() == AArch64::BLRCall); TailOpcode = AArch64::TCRETURNriALL; } MachineInstr *TC = BuildMI(MF, DebugLoc(), get(TailOpcode)) Index: llvm/lib/Target/AArch64/AArch64InstrInfo.td =================================================================== --- llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -588,6 +588,8 @@ def UseBTI : Predicate<[{ MF->getFunction().hasFnAttribute("branch-target-enforcement") }]>; def NotUseBTI : Predicate<[{ !MF->getFunction().hasFnAttribute("branch-target-enforcement") }]>; + def SLSBLRMitigationWithBTI : Predicate<[{ (MF->getFunction().hasFnAttribute("branch-target-enforcement") && MF->getSubtarget().hardenSlsBlr()) }]>; + def NotSLSBLRMitigationWithBTI : Predicate<[{ !(MF->getFunction().hasFnAttribute("branch-target-enforcement") && MF->getSubtarget().hardenSlsBlr()) }]>; // Toggles patterns which aren't beneficial in GlobalISel when we aren't // optimizing. This allows us to selectively use patterns without impacting @@ -2014,9 +2016,20 @@ def : InstAlias<"ret", (RET LR)>; let isCall = 1, Defs = [LR], Uses = [SP] in { -def BLR : BranchReg<0b0001, "blr", [(AArch64call GPR64:$Rn)]>; + def BLR : BranchReg<0b0001, "blr", []>; + def BLRCall : Pseudo<(outs), (ins GPR64:$Rn), []>, + Sched<[WriteBrReg]>; + def BLRCallX16X17 : Pseudo<(outs), (ins rtcGPR64:$Rn), []>, + Sched<[WriteBrReg]>; } // isCall +def : Pat<(AArch64call GPR64:$Rn), + (BLRCall GPR64:$Rn)>, + Requires<[NotSLSBLRMitigationWithBTI]>; +def : Pat<(AArch64call rtcGPR64:$Rn), + (BLRCallX16X17 rtcGPR64:$Rn)>, + Requires<[SLSBLRMitigationWithBTI]>; + let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in { def BR : BranchReg<0b0000, "br", [(brind GPR64:$Rn)]>; } // isBranch, isTerminator, isBarrier, isIndirectBranch Index: llvm/lib/Target/AArch64/AArch64RegisterBankInfo.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64RegisterBankInfo.cpp +++ llvm/lib/Target/AArch64/AArch64RegisterBankInfo.cpp @@ -261,6 +261,7 @@ case AArch64::GPR64common_and_GPR64noipRegClassID: case AArch64::GPR64noip_and_tcGPR64RegClassID: case AArch64::tcGPR64RegClassID: + case AArch64::rtcGPR64RegClassID: case AArch64::WSeqPairsClassRegClassID: case AArch64::XSeqPairsClassRegClassID: return getRegBank(AArch64::GPRRegBankID); Index: llvm/lib/Target/AArch64/AArch64SLSHardening.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64SLSHardening.cpp +++ llvm/lib/Target/AArch64/AArch64SLSHardening.cpp @@ -108,8 +108,11 @@ static bool isBLR(const MachineInstr &MI) { switch (MI.getOpcode()) { - case AArch64::BLR: + case AArch64::BLRCall: + case AArch64::BLRCallX16X17: return true; + case AArch64::BLR: + llvm_unreachable("BLR should only be used at the MC layer level."); case AArch64::BLRAA: case AArch64::BLRAB: case AArch64::BLRAAZ: @@ -264,11 +267,14 @@ Register Reg; bool RegIsKilled; switch (BLR.getOpcode()) { - case AArch64::BLR: + case AArch64::BLRCall: + case AArch64::BLRCallX16X17: BLOpcode = AArch64::BL; Reg = BLR.getOperand(0).getReg(); RegIsKilled = BLR.getOperand(0).isKill(); break; + case AArch64::BLR: + llvm_unreachable("BLR should only be used at the MC layer level."); case AArch64::BLRAA: case AArch64::BLRAB: case AArch64::BLRAAZ: Index: llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp +++ llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp @@ -776,7 +776,7 @@ static unsigned getCallOpcode(const Function &CallerF, bool IsIndirect, bool IsTailCall) { if (!IsTailCall) - return IsIndirect ? AArch64::BLR : AArch64::BL; + return IsIndirect ? AArch64::BLRCall : AArch64::BL; if (!IsIndirect) return AArch64::TCRETURNdi; Index: llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp =================================================================== --- llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -2871,7 +2871,7 @@ // TLS calls preserve all registers except those that absolutely must be // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be // silly). - MIB.buildInstr(AArch64::BLR, {}, {Load}) + MIB.buildInstr(AArch64::BLRCall, {}, {Load}) .addDef(AArch64::X0, RegState::Implicit) .addRegMask(TRI.getTLSCallPreservedMask()); Index: llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll +++ llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll @@ -37,7 +37,7 @@ ; Make sure the register feeding the indirect call is properly constrained. ; CHECK: - { id: [[FUNC:[0-9]+]], class: gpr64, preferred-register: '' } ; CHECK: %[[FUNC]]:gpr64(p0) = COPY $x0 -; CHECK: BLR %[[FUNC]](p0), csr_aarch64_aapcs, implicit-def $lr, implicit $sp +; CHECK: BLRCall %[[FUNC]](p0), csr_aarch64_aapcs, implicit-def $lr, implicit $sp ; CHECK: RET_ReallyLR define void @test_indirect_call(void()* %func) { call void %func() Index: llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll +++ llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll @@ -45,7 +45,7 @@ ; CHECK-LABEL: name: test_invoke_indirect ; CHECK: [[CALLEE:%[0-9]+]]:gpr64(p0) = COPY $x0 -; CHECK: BLR [[CALLEE]] +; CHECK: BLRCall [[CALLEE]] define void @test_invoke_indirect(void()* %callee) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { invoke void %callee() to label %continue unwind label %broken Index: llvm/test/CodeGen/AArch64/chkstk.ll =================================================================== --- llvm/test/CodeGen/AArch64/chkstk.ll +++ llvm/test/CodeGen/AArch64/chkstk.ll @@ -30,4 +30,4 @@ ; CHECK-LARGE-CODE-MODEL: blr x16 ; CHECK-LARGE-CODE-MODEL: sub sp, sp, x15, lsl #4 -; CHECK-REGSTATE-LARGE: frame-setup BLR killed $x16, implicit-def $lr, implicit $sp, implicit-def $x15, implicit-def dead $x16, implicit-def dead $x17, implicit-def dead $nzcv +; CHECK-REGSTATE-LARGE: frame-setup BLRCallX16X17 killed $x16, implicit-def $lr, implicit $sp, implicit-def $x15, implicit-def dead $x16, implicit-def dead $x17, implicit-def dead $nzcv Index: llvm/test/CodeGen/AArch64/speculation-hardening-sls-blr.mir =================================================================== --- llvm/test/CodeGen/AArch64/speculation-hardening-sls-blr.mir +++ llvm/test/CodeGen/AArch64/speculation-hardening-sls-blr.mir @@ -37,7 +37,7 @@ frame-setup CFI_INSTRUCTION offset $w30, -16 renamable $x8 = ADRP target-flags(aarch64-page) @a renamable $x8 = LDRXui killed renamable $x8, target-flags(aarch64-pageoff, aarch64-nc) @a :: (dereferenceable load 8 from `i32 ()** bitcast (i32 (...)** @a to i32 ()**)`) - BLR killed renamable $x8, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $w0 + BLRCall killed renamable $x8, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $w0 ; CHECK: BL , csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $w0, implicit killed $x8 renamable $x8 = ADRP target-flags(aarch64-page) @b STRWui killed renamable $w0, killed renamable $x8, target-flags(aarch64-pageoff, aarch64-nc) @b :: (store 4 into @b) Index: llvm/test/CodeGen/AArch64/speculation-hardening-sls.ll =================================================================== --- llvm/test/CodeGen/AArch64/speculation-hardening-sls.ll +++ llvm/test/CodeGen/AArch64/speculation-hardening-sls.ll @@ -2,6 +2,7 @@ ; RUN: llc -mattr=harden-sls-retbr,harden-sls-blr -mattr=+sb -verify-machineinstrs -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,SB --dump-input-on-failure ; RUN: llc -global-isel -global-isel-abort=0 -mattr=harden-sls-retbr,harden-sls-blr -verify-machineinstrs -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,ISBDSB --dump-input-on-failure ; RUN: llc -global-isel -global-isel-abort=0 -mattr=harden-sls-retbr,harden-sls-blr -mattr=+sb -verify-machineinstrs -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,SB --dump-input-on-failure +; RUN: llc -mattr=+sb -mattr=harden-sls-blr -verify-machineinstrs -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s --check-prefixes=BLRBTISB --dump-input-on-failure ; Function Attrs: norecurse nounwind readnone define dso_local i32 @double_return(i32 %a, i32 %b) { @@ -88,32 +89,39 @@ } define dso_local i32 @indirect_call( -i32 (...)* nocapture %f1, i32 (...)* nocapture %f2) { +i32 (...)* nocapture %f1, i32 (...)* nocapture %f2) "branch-target-enforcement" { entry: ; CHECK-LABEL: indirect_call: +; BLRBTISB-LABEL: indirect_call: %callee.knr.cast = bitcast i32 (...)* %f1 to i32 ()* %call = tail call i32 %callee.knr.cast() ; CHECK: bl {{__llvm_slsblr_thunk_x[0-9]+$}} +; BLRBTISB: bl {{__llvm_slsblr_thunk_x(16|17)$}} %callee.knr.cast1 = bitcast i32 (...)* %f2 to i32 ()* %call2 = tail call i32 %callee.knr.cast1() ; CHECK: bl {{__llvm_slsblr_thunk_x[0-9]+$}} +; BLRBTISB: bl {{__llvm_slsblr_thunk_x(16|17)$}} %add = add nsw i32 %call2, %call ret i32 %add ; CHECK: .Lfunc_end +; BLRBTISB: .Lfunc_end } ; verify calling through a function pointer. @a = dso_local local_unnamed_addr global i32 (...)* null, align 8 @b = dso_local local_unnamed_addr global i32 0, align 4 -define dso_local void @indirect_call_global() local_unnamed_addr { +define dso_local void @indirect_call_global() "branch-target-enforcement" { ; CHECK-LABEL: indirect_call_global: +; CLRBTISB-LABEL: indirect_call_global: entry: %0 = load i32 ()*, i32 ()** bitcast (i32 (...)** @a to i32 ()**), align 8 %call = tail call i32 %0() nounwind ; CHECK: bl {{__llvm_slsblr_thunk_x[0-9]+$}} +; BLRBTISB: bl {{__llvm_slsblr_thunk_x(16|17)$}} store i32 %call, i32* @b, align 4 ret void ; CHECK: .Lfunc_end +; BLRBTISB: .Lfunc_end } ; CHECK-label: __llvm_slsblr_thunk_x0: