Index: include/llvm/Bitcode/LLVMBitCodes.h =================================================================== --- include/llvm/Bitcode/LLVMBitCodes.h +++ include/llvm/Bitcode/LLVMBitCodes.h @@ -561,6 +561,7 @@ ATTR_KIND_SPECULATABLE = 53, ATTR_KIND_STRICT_FP = 54, ATTR_KIND_SANITIZE_HWADDRESS = 55, + ATTR_KIND_NOCF_CHECK = 56, }; enum ComdatSelectionKindCodes { Index: include/llvm/CodeGen/TargetLowering.h =================================================================== --- include/llvm/CodeGen/TargetLowering.h +++ include/llvm/CodeGen/TargetLowering.h @@ -3530,6 +3530,13 @@ virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const; + /// Expands target specific indirect branch for the case of JumpTable + /// expanasion. + virtual SDValue expandIndirectJTBranch(const SDLoc& dl, SDValue Value, SDValue Addr, + SelectionDAG &DAG) const { + return DAG.getNode(ISD::BRIND, dl, MVT::Other, Value, Addr); + } + // seteq(x, 0) -> truncate(srl(ctlz(zext(x)), log2(#bits))) // If we're comparing for equality to zero and isCtlzFast is true, expose the // fact that this can be implemented as a ctlz/srl pair, so that the dag Index: include/llvm/IR/Attributes.td =================================================================== --- include/llvm/IR/Attributes.td +++ include/llvm/IR/Attributes.td @@ -106,6 +106,9 @@ /// Mark the function as not returning. def NoReturn : EnumAttr<"noreturn">; +/// Disable Indirect Branch Tracking. +def NoCfCheck : EnumAttr<"nocf_check">; + /// Function doesn't unwind stack. def NoUnwind : EnumAttr<"nounwind">; Index: include/llvm/IR/Function.h =================================================================== --- include/llvm/IR/Function.h +++ include/llvm/IR/Function.h @@ -462,6 +462,9 @@ addFnAttr(Attribute::NoReturn); } + /// Determine if the function should not perform indirect branch tracking. + bool doesNoCfCheck() const { return hasFnAttribute(Attribute::NoCfCheck); } + /// @brief Determine if the function cannot unwind. bool doesNotThrow() const { return hasFnAttribute(Attribute::NoUnwind); Index: include/llvm/IR/Instructions.h =================================================================== --- include/llvm/IR/Instructions.h +++ include/llvm/IR/Instructions.h @@ -1831,6 +1831,9 @@ addAttribute(AttributeList::FunctionIndex, Attribute::NoReturn); } + /// Determine if the call should not perform indirect branch tracking. + bool doesNoCfCheck() const { return hasFnAttr(Attribute::NoCfCheck); } + /// Determine if the call cannot unwind. bool doesNotThrow() const { return hasFnAttr(Attribute::NoUnwind); } void setDoesNotThrow() { @@ -3931,6 +3934,9 @@ addAttribute(AttributeList::FunctionIndex, Attribute::NoReturn); } + /// Determine if the call should not perform indirect branch tracking. + bool doesNoCfCheck() const { return hasFnAttr(Attribute::NoCfCheck); } + /// Determine if the call cannot unwind. bool doesNotThrow() const { return hasFnAttr(Attribute::NoUnwind); } void setDoesNotThrow() { Index: lib/AsmParser/LLLexer.cpp =================================================================== --- lib/AsmParser/LLLexer.cpp +++ lib/AsmParser/LLLexer.cpp @@ -648,6 +648,7 @@ KEYWORD(nonnull); KEYWORD(noredzone); KEYWORD(noreturn); + KEYWORD(nocf_check); KEYWORD(nounwind); KEYWORD(optnone); KEYWORD(optsize); Index: lib/AsmParser/LLParser.cpp =================================================================== --- lib/AsmParser/LLParser.cpp +++ lib/AsmParser/LLParser.cpp @@ -1130,6 +1130,7 @@ case lltok::kw_nonlazybind: B.addAttribute(Attribute::NonLazyBind); break; case lltok::kw_noredzone: B.addAttribute(Attribute::NoRedZone); break; case lltok::kw_noreturn: B.addAttribute(Attribute::NoReturn); break; + case lltok::kw_nocf_check: B.addAttribute(Attribute::NoCfCheck); break; case lltok::kw_norecurse: B.addAttribute(Attribute::NoRecurse); break; case lltok::kw_nounwind: B.addAttribute(Attribute::NoUnwind); break; case lltok::kw_optnone: B.addAttribute(Attribute::OptimizeNone); break; @@ -1467,6 +1468,7 @@ case lltok::kw_nonlazybind: case lltok::kw_noredzone: case lltok::kw_noreturn: + case lltok::kw_nocf_check: case lltok::kw_nounwind: case lltok::kw_optnone: case lltok::kw_optsize: @@ -1560,6 +1562,7 @@ case lltok::kw_nonlazybind: case lltok::kw_noredzone: case lltok::kw_noreturn: + case lltok::kw_nocf_check: case lltok::kw_nounwind: case lltok::kw_optnone: case lltok::kw_optsize: Index: lib/AsmParser/LLToken.h =================================================================== --- lib/AsmParser/LLToken.h +++ lib/AsmParser/LLToken.h @@ -199,6 +199,7 @@ kw_nonnull, kw_noredzone, kw_noreturn, + kw_nocf_check, kw_nounwind, kw_optnone, kw_optsize, Index: lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- lib/Bitcode/Reader/BitcodeReader.cpp +++ lib/Bitcode/Reader/BitcodeReader.cpp @@ -1157,6 +1157,7 @@ case Attribute::Speculatable: return 1ULL << 54; case Attribute::StrictFP: return 1ULL << 55; case Attribute::SanitizeHWAddress: return 1ULL << 56; + case Attribute::NoCfCheck: return 1ULL << 57; case Attribute::Dereferenceable: llvm_unreachable("dereferenceable attribute not supported in raw format"); break; @@ -1335,6 +1336,8 @@ return Attribute::NoRedZone; case bitc::ATTR_KIND_NO_RETURN: return Attribute::NoReturn; + case bitc::ATTR_KIND_NOCF_CHECK: + return Attribute::NoCfCheck; case bitc::ATTR_KIND_NO_UNWIND: return Attribute::NoUnwind; case bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE: Index: lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- lib/Bitcode/Writer/BitcodeWriter.cpp +++ lib/Bitcode/Writer/BitcodeWriter.cpp @@ -635,6 +635,8 @@ return bitc::ATTR_KIND_NO_RED_ZONE; case Attribute::NoReturn: return bitc::ATTR_KIND_NO_RETURN; + case Attribute::NoCfCheck: + return bitc::ATTR_KIND_NOCF_CHECK; case Attribute::NoUnwind: return bitc::ATTR_KIND_NO_UNWIND; case Attribute::OptimizeForSize: Index: lib/CodeGen/SelectionDAG/LegalizeDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -3699,7 +3699,8 @@ Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, TLI.getPICJumpTableRelocBase(Table, DAG)); } - Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr); + + Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG); Results.push_back(Tmp1); break; } Index: lib/IR/Attributes.cpp =================================================================== --- lib/IR/Attributes.cpp +++ lib/IR/Attributes.cpp @@ -299,6 +299,8 @@ return "noredzone"; if (hasAttribute(Attribute::NoReturn)) return "noreturn"; + if (hasAttribute(Attribute::NoCfCheck)) + return "nocf_check"; if (hasAttribute(Attribute::NoRecurse)) return "norecurse"; if (hasAttribute(Attribute::NoUnwind)) Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -1381,6 +1381,7 @@ static bool isFuncOnlyAttr(Attribute::AttrKind Kind) { switch (Kind) { case Attribute::NoReturn: + case Attribute::NoCfCheck: case Attribute::NoUnwind: case Attribute::NoInline: case Attribute::AlwaysInline: Index: lib/Target/X86/AsmParser/X86AsmParser.cpp =================================================================== --- lib/Target/X86/AsmParser/X86AsmParser.cpp +++ lib/Target/X86/AsmParser/X86AsmParser.cpp @@ -2359,21 +2359,22 @@ .Cases("acquire", "release", isParsingIntelSyntax()) .Default(false); - auto isLockRepeatPrefix = [](StringRef N) { + auto isLockRepeatNtPrefix = [](StringRef N) { return StringSwitch(N) - .Cases("lock", "rep", "repe", "repz", "repne", "repnz", true) + .Cases("lock", "rep", "repe", "repz", "repne", "repnz", "notrack", true) .Default(false); }; bool CurlyAsEndOfStatement = false; unsigned Flags = X86::IP_NO_PREFIX; - while (isLockRepeatPrefix(Name.lower())) { + while (isLockRepeatNtPrefix(Name.lower())) { unsigned Prefix = StringSwitch(Name) .Cases("lock", "lock", X86::IP_HAS_LOCK) .Cases("rep", "repe", "repz", X86::IP_HAS_REPEAT) .Cases("repne", "repnz", X86::IP_HAS_REPEAT_NE) + .Cases("notrack", "notrack", X86::IP_HAS_NOTRACK) .Default(X86::IP_NO_PREFIX); // Invalid prefix (impossible) Flags |= Prefix; if (getLexer().is(AsmToken::EndOfStatement)) { Index: lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp =================================================================== --- lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp +++ lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp @@ -56,6 +56,9 @@ if (!(TSFlags & X86II::LOCK) && Flags & X86::IP_HAS_LOCK) OS << "\tlock\t"; + if ((TSFlags & X86II::NOTRACK) || (Flags & X86::IP_HAS_NOTRACK)) + OS << "\tnotrack\t"; + if (Flags & X86::IP_HAS_REPEAT_NE) OS << "\trepne\t"; else if (Flags & X86::IP_HAS_REPEAT) Index: lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp =================================================================== --- lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp +++ lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp @@ -49,6 +49,9 @@ else if (Flags & X86::IP_HAS_REPEAT) OS << "\trep\t"; + if ((TSFlags & X86II::NOTRACK) || (Flags & X86::IP_HAS_NOTRACK)) + OS << "\tnotrack\t"; + printInstruction(MI, OS); // Next always print the annotation. Index: lib/Target/X86/MCTargetDesc/X86BaseInfo.h =================================================================== --- lib/Target/X86/MCTargetDesc/X86BaseInfo.h +++ lib/Target/X86/MCTargetDesc/X86BaseInfo.h @@ -60,8 +60,9 @@ IP_HAS_REPEAT_NE = 4, IP_HAS_REPEAT = 8, IP_HAS_LOCK = 16, - NO_SCHED_INFO = 32 // Don't add sched comment to the current instr because - // it was already added + NO_SCHED_INFO = 32, // Don't add sched comment to the current instr because + // it was already added + IP_HAS_NOTRACK = 64 }; } // end namespace X86; @@ -572,7 +573,11 @@ /// Explicitly specified rounding control EVEX_RCShift = Has3DNow0F0FOpcodeShift + 1, - EVEX_RC = 1ULL << EVEX_RCShift + EVEX_RC = 1ULL << EVEX_RCShift, + + // NOTRACK prefix + NoTrackShift = EVEX_RCShift + 1, + NOTRACK = 1ULL << NoTrackShift }; // getBaseOpcodeFor - This function returns the "base" X86 opcode for the Index: lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp =================================================================== --- lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp +++ lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp @@ -1111,6 +1111,10 @@ if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK) EmitByte(0xF0, CurByte, OS); + // Emit the NOTRACK opcode prefix. + if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK) + EmitByte(0x3E, CurByte, OS); + switch (TSFlags & X86II::OpPrefixMask) { case X86II::PD: // 66 EmitByte(0x66, CurByte, OS); Index: lib/Target/X86/X86FastISel.cpp =================================================================== --- lib/Target/X86/X86FastISel.cpp +++ lib/Target/X86/X86FastISel.cpp @@ -3167,6 +3167,13 @@ CLI.CS ? dyn_cast(CLI.CS->getInstruction()) : nullptr; const Function *CalledFn = CI ? CI->getCalledFunction() : nullptr; + // Call / invoke instructions with NoCfCheck attribute require special + // handling. + const auto *II = + CLI.CS ? dyn_cast(CLI.CS->getInstruction()) : nullptr; + if ((CI && CI->doesNoCfCheck()) || (II && II->doesNoCfCheck())) + return false; + // Functions with no_caller_saved_registers that need special handling. if ((CI && CI->hasFnAttr("no_caller_saved_registers")) || (CalledFn && CalledFn->hasFnAttribute("no_caller_saved_registers"))) Index: lib/Target/X86/X86ISelLowering.h =================================================================== --- lib/Target/X86/X86ISelLowering.h +++ lib/Target/X86/X86ISelLowering.h @@ -75,6 +75,9 @@ /// CALL, + /// Same as call except it adds the NoTrack prefix. + NT_CALL, + /// This operation implements the lowering for readcyclecounter. RDTSC_DAG, @@ -122,6 +125,10 @@ /// or TEST instruction. BRCOND, + /// BRIND node with NoTrack prefix. Operand 0 is the chain operand and + /// operand 1 is the target address. + NT_BRIND, + /// Return with a flag operand. Operand 0 is the chain operand, operand /// 1 is the number of bytes of stack to pop. RET_FLAG, @@ -1098,9 +1105,12 @@ bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override; - void finalizeLowering(MachineFunction &MF) const override; + SDValue expandIndirectJTBranch(const SDLoc& dl, SDValue Value, + SDValue Addr, SelectionDAG &DAG) + const override; + protected: std::pair findRepresentativeClass(const TargetRegisterInfo *TRI, Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -3355,6 +3355,11 @@ const Function *Fn = CI ? CI->getCalledFunction() : nullptr; bool HasNCSR = (CI && CI->hasFnAttr("no_caller_saved_registers")) || (Fn && Fn->hasFnAttribute("no_caller_saved_registers")); + const auto *II = dyn_cast_or_null(CLI.CS.getInstruction()); + bool HasNoCfCheck = + (CI && CI->doesNoCfCheck()) || (II && II->doesNoCfCheck()); + const Module *M = MF.getMMI().getModule(); + Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch"); if (CallConv == CallingConv::X86_INTR) report_fatal_error("X86 interrupts may not be called directly"); @@ -3830,7 +3835,11 @@ return DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops); } - Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops); + if (HasNoCfCheck && IsCFProtectionSupported) { + Chain = DAG.getNode(X86ISD::NT_CALL, dl, NodeTys, Ops); + } else { + Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops); + } InFlag = Chain.getValue(1); // Create the CALLSEQ_END node. @@ -25492,6 +25501,8 @@ case X86ISD::GF2P8MULB: return "X86ISD::GF2P8MULB"; case X86ISD::GF2P8AFFINEQB: return "X86ISD::GF2P8AFFINEQB"; case X86ISD::GF2P8AFFINEINVQB: return "X86ISD::GF2P8AFFINEINVQB"; + case X86ISD::NT_CALL: return "X86ISD::NT_CALL"; + case X86ISD::NT_BRIND: return "X86ISD::NT_BRIND"; } return nullptr; } @@ -37852,6 +37863,22 @@ TargetLoweringBase::finalizeLowering(MF); } +SDValue X86TargetLowering::expandIndirectJTBranch(const SDLoc& dl, + SDValue Value, SDValue Addr, + SelectionDAG &DAG) const { + const Module *M = DAG.getMachineFunction().getMMI().getModule(); + Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch"); + if (IsCFProtectionSupported) { + // In case control-flow branch protection is enabled, we need to add + // notrack prefix to the indirect branch. + // In order to do that we create NT_BRIND SDNode. + // Upon ISEL, the pattern will convert it to jmp with NoTrack prefix. + return DAG.getNode(X86ISD::NT_BRIND, dl, MVT::Other, Value, Addr); + } + + return TargetLowering::expandIndirectJTBranch(dl, Value, Addr, DAG); +} + /// This method query the target whether it is beneficial for dag combiner to /// promote the specified node. If true, it should return the desired promotion /// type by reference. Index: lib/Target/X86/X86IndirectBranchTracking.cpp =================================================================== --- lib/Target/X86/X86IndirectBranchTracking.cpp +++ lib/Target/X86/X86IndirectBranchTracking.cpp @@ -22,7 +22,6 @@ #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" -#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" using namespace llvm; @@ -55,21 +54,11 @@ /// Endbr opcode for the current machine function. unsigned int EndbrOpcode; - /// The function looks for an indirect jump terminator in MBB predecessors. - /// - /// Jump tables are generated when lowering switch-case statements or - /// setjmp/longjump functions. - /// As a result only indirect jumps use jump tables. - /// The function verifies this assumption. - /// - /// \return true if the input \p MBB has a predecessor MBB with indirect - /// branch terminator or false otherwise. - bool verifyIndirectJump(const MachineBasicBlock *MBB) const; - /// Adds a new ENDBR instruction to the begining of the MBB. /// The function will not add it if already exists. /// It will add ENDBR32 or ENDBR64 opcode, depending on the target. - void addENDBR(MachineBasicBlock &MBB) const; + /// \returns true if the ENDBR was added and false otherwise. + bool addENDBR(MachineBasicBlock &MBB) const; }; } // end anonymous namespace @@ -80,17 +69,7 @@ return new X86IndirectBranchTrackingPass(); } -bool X86IndirectBranchTrackingPass::verifyIndirectJump( - const MachineBasicBlock *MBB) const { - for (auto &PredMBB : MBB->predecessors()) - for (auto &TermI : PredMBB->terminators()) - if (TermI.isIndirectBranch()) - return true; - - return false; -} - -void X86IndirectBranchTrackingPass::addENDBR(MachineBasicBlock &MBB) const { +bool X86IndirectBranchTrackingPass::addENDBR(MachineBasicBlock &MBB) const { assert(TII && "Target instruction info was not initialized"); assert((X86::ENDBR64 == EndbrOpcode || X86::ENDBR32 == EndbrOpcode) && "Unexpected Endbr opcode"); @@ -101,13 +80,16 @@ if (MI == MBB.end() || EndbrOpcode != MI->getOpcode()) { BuildMI(MBB, MI, MBB.findDebugLoc(MI), TII->get(EndbrOpcode)); NumEndBranchAdded++; + return true; } + + return false; } bool X86IndirectBranchTrackingPass::runOnMachineFunction(MachineFunction &MF) { const X86Subtarget &SubTarget = MF.getSubtarget(); - // Make sure that the target supports ENDBR instruction. + // Make sure that the target supports IBT instruction. if (!SubTarget.hasIBT()) return false; @@ -124,40 +106,20 @@ EndbrOpcode = SubTarget.is64Bit() ? X86::ENDBR64 : X86::ENDBR32; // Non-internal function or function whose address was taken, can be - // invoked through indirect calls. Mark the first BB with ENDBR instruction. - // TODO: Do not add ENDBR instruction in case notrack attribute is used. - if (MF.getFunction().hasAddressTaken() || - !MF.getFunction().hasLocalLinkage()) { + // accessed through indirect calls. Mark the first BB with ENDBR instruction + // unless nocf_check attribute is used. + if ((MF.getFunction().hasAddressTaken() || + !MF.getFunction().hasLocalLinkage()) && + !MF.getFunction().doesNoCfCheck()) { auto MBB = MF.begin(); - addENDBR(*MBB); - Changed = true; + Changed |= addENDBR(*MBB); } - for (auto &MBB : MF) { - // Find all basic blocks that thier address was taken (for example + for (auto &MBB : MF) + // Find all basic blocks that their address was taken (for example // in the case of indirect jump) and add ENDBR instruction. - if (MBB.hasAddressTaken()) { - addENDBR(MBB); - Changed = true; - } - } - - // Adds ENDBR instructions to MBB destinations of the jump table. - // TODO: In case of more than 50 destinations, do not add ENDBR and - // instead add DS_PREFIX. - if (MachineJumpTableInfo *JTI = MF.getJumpTableInfo()) { - for (const auto &JT : JTI->getJumpTables()) { - for (auto *MBB : JT.MBBs) { - // This assert verifies the assumption that this MBB has an indirect - // jump terminator in one of its predecessor. - assert(verifyIndirectJump(MBB) && - "The MBB is not the destination of an indirect jump"); - - addENDBR(*MBB); - Changed = true; - } - } - } + if (MBB.hasAddressTaken()) + Changed |= addENDBR(MBB); return Changed; } Index: lib/Target/X86/X86InstrControl.td =================================================================== --- lib/Target/X86/X86InstrControl.td +++ lib/Target/X86/X86InstrControl.td @@ -153,6 +153,30 @@ [(brind (loadi64 addr:$dst))], IIC_JMP_MEM>, Requires<[In64BitMode]>, Sched<[WriteJumpLd]>; + let isCodeGenOnly = 1, Predicates = [HasIBT] in { + def NT_JMP16r : I<0xFF, MRM4r, (outs), (ins GR16 : $dst), "jmp{w}\t{*}$dst", + [(X86NoTrackBrind GR16 : $dst)], IIC_JMP_REG>, Requires<[Not64BitMode]>, + OpSize16, Sched<[WriteJump]>, NOTRACK; + + def NT_JMP16m : I<0xFF, MRM4m, (outs), (ins i16mem : $dst), "jmp{w}\t{*}$dst", + [(X86NoTrackBrind (loadi16 addr : $dst))], IIC_JMP_MEM>, + Requires<[Not64BitMode]>, OpSize16, Sched<[WriteJumpLd]>, NOTRACK; + + def NT_JMP32r : I<0xFF, MRM4r, (outs), (ins GR32 : $dst), "jmp{l}\t{*}$dst", + [(X86NoTrackBrind GR32 : $dst)], IIC_JMP_REG>, Requires<[Not64BitMode]>, + OpSize32, Sched<[WriteJump]>, NOTRACK; + def NT_JMP32m : I<0xFF, MRM4m, (outs), (ins i32mem : $dst), "jmp{l}\t{*}$dst", + [(X86NoTrackBrind (loadi32 addr : $dst))], IIC_JMP_MEM>, + Requires<[Not64BitMode]>, OpSize32, Sched<[WriteJumpLd]>, NOTRACK; + + def NT_JMP64r : I<0xFF, MRM4r, (outs), (ins GR64 : $dst), "jmp{q}\t{*}$dst", + [(X86NoTrackBrind GR64 : $dst)], IIC_JMP_REG>, Requires<[In64BitMode]>, + Sched<[WriteJump]>, NOTRACK; + def NT_JMP64m : I<0xFF, MRM4m, (outs), (ins i64mem : $dst), "jmp{q}\t{*}$dst", + [(X86NoTrackBrind(loadi64 addr : $dst))], IIC_JMP_MEM>, + Requires<[In64BitMode]>, Sched<[WriteJumpLd]>, NOTRACK; + } + let Predicates = [Not64BitMode] in { def FARJMP16i : Iseg16<0xEA, RawFrmImm16, (outs), (ins i16imm:$off, i16imm:$seg), @@ -175,7 +199,6 @@ Sched<[WriteJumpLd]>; } - // Loop instructions let SchedRW = [WriteJump] in { def LOOP : Ii8PCRel<0xE2, RawFrm, (outs), (ins brtarget8:$dst), "loop\t$dst", [], IIC_LOOP>; @@ -218,6 +241,25 @@ Requires<[Not64BitMode,FavorMemIndirectCall]>, Sched<[WriteJumpLd]>; + let isCodeGenOnly = 1, Predicates = [HasIBT] in { + def NT_CALL16r : I<0xFF, MRM2r, (outs), (ins GR16 : $dst), + "call{w}\t{*}$dst",[(X86NoTrackCall GR16 : $dst)], IIC_CALL_RI>, + OpSize16, Requires<[Not64BitMode]>, Sched<[WriteJump]>, NOTRACK; + def NT_CALL16m : I<0xFF, MRM2m, (outs), (ins i16mem : $dst), + "call{w}\t{*}$dst",[(X86NoTrackCall(loadi16 addr : $dst))], + IIC_CALL_MEM>, OpSize16, + Requires<[Not64BitMode,FavorMemIndirectCall]>, + Sched<[WriteJumpLd]>, NOTRACK; + def NT_CALL32r : I<0xFF, MRM2r, (outs), (ins GR32 : $dst), + "call{l}\t{*}$dst",[(X86NoTrackCall GR32 : $dst)], IIC_CALL_RI>, + OpSize32, Requires<[Not64BitMode]>, Sched<[WriteJump]>, NOTRACK; + def NT_CALL32m : I<0xFF, MRM2m, (outs), (ins i32mem : $dst), + "call{l}\t{*}$dst",[(X86NoTrackCall(loadi32 addr : $dst))], + IIC_CALL_MEM>, OpSize32, + Requires<[Not64BitMode,FavorMemIndirectCall]>, + Sched<[WriteJumpLd]>, NOTRACK; + } + let Predicates = [Not64BitMode] in { def FARCALL16i : Iseg16<0x9A, RawFrmImm16, (outs), (ins i16imm:$off, i16imm:$seg), @@ -304,6 +346,17 @@ IIC_CALL_MEM>, Requires<[In64BitMode,FavorMemIndirectCall]>; + let isCodeGenOnly = 1, Predicates = [HasIBT] in{ + def NT_CALL64r : I<0xFF, MRM2r, (outs), (ins GR64 : $dst), + "call{q}\t{*}$dst",[(X86NoTrackCall GR64 : $dst)], + IIC_CALL_RI>, + Requires<[In64BitMode]>, NOTRACK; + def NT_CALL64m : I<0xFF, MRM2m, (outs), (ins i64mem : $dst), + "call{q}\t{*}$dst",[(X86NoTrackCall(loadi64 addr : $dst))], + IIC_CALL_MEM>, + Requires<[In64BitMode,FavorMemIndirectCall]>, NOTRACK; + } + def FARCALL64 : RI<0xFF, MRM3m, (outs), (ins opaque80mem:$dst), "lcall{q}\t{*}$dst", [], IIC_CALL_FAR_MEM>; } Index: lib/Target/X86/X86InstrFormats.td =================================================================== --- lib/Target/X86/X86InstrFormats.td +++ lib/Target/X86/X86InstrFormats.td @@ -215,6 +215,7 @@ class EVEX_V512 { bit hasEVEX_L2 = 1; bit hasVEX_L = 0; } class EVEX_V256 { bit hasEVEX_L2 = 0; bit hasVEX_L = 1; } class EVEX_V128 { bit hasEVEX_L2 = 0; bit hasVEX_L = 0; } +class NOTRACK { bit hasNoTrackPrefix = 1; } // Specify AVX512 8-bit compressed displacement encoding based on the vector // element size in bits (8, 16, 32, 64) and the CDisp8 form. @@ -296,6 +297,7 @@ int CD8_EltSize = 0; // Compressed disp8 form - element-size in bytes. bit has3DNow0F0FOpcode =0;// Wacky 3dNow! encoding? bit hasEVEX_RC = 0; // Explicitly specified rounding control in FP instruction. + bit hasNoTrackPrefix = 0; // Does this inst has 0x3E (NoTrack) prefix? bits<2> EVEX_LL; let EVEX_LL{0} = hasVEX_L; @@ -347,6 +349,7 @@ let TSFlags{52-46} = CD8_Scale; let TSFlags{53} = has3DNow0F0FOpcode; let TSFlags{54} = hasEVEX_RC; + let TSFlags{55} = hasNoTrackPrefix; } class PseudoI pattern, Index: lib/Target/X86/X86InstrInfo.td =================================================================== --- lib/Target/X86/X86InstrInfo.td +++ lib/Target/X86/X86InstrInfo.td @@ -94,6 +94,8 @@ def SDT_X86Call : SDTypeProfile<0, -1, [SDTCisVT<0, iPTR>]>; +def SDT_X86NtBrind : SDTypeProfile<0, -1, [SDTCisVT<0, iPTR>]>; + def SDT_X86VASTART_SAVE_XMM_REGS : SDTypeProfile<0, -1, [SDTCisVT<0, i8>, SDTCisVT<1, iPTR>, SDTCisVT<2, iPTR>]>; @@ -196,6 +198,12 @@ [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue, SDNPVariadic]>; +def X86NoTrackCall : SDNode<"X86ISD::NT_CALL", SDT_X86Call, + [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue, + SDNPVariadic]>; +def X86NoTrackBrind : SDNode<"X86ISD::NT_BRIND", SDT_X86NtBrind, + [SDNPHasChain]>; + def X86rep_stos: SDNode<"X86ISD::REP_STOS", SDTX86RepStr, [SDNPHasChain, SDNPInGlue, SDNPOutGlue, SDNPMayStore]>; def X86rep_movs: SDNode<"X86ISD::REP_MOVS", SDTX86RepStr, Index: lib/Transforms/IPO/ForceFunctionAttrs.cpp =================================================================== --- lib/Transforms/IPO/ForceFunctionAttrs.cpp +++ lib/Transforms/IPO/ForceFunctionAttrs.cpp @@ -42,6 +42,7 @@ .Case("nonlazybind", Attribute::NonLazyBind) .Case("noredzone", Attribute::NoRedZone) .Case("noreturn", Attribute::NoReturn) + .Case("nocf_check", Attribute::NoCfCheck) .Case("norecurse", Attribute::NoRecurse) .Case("nounwind", Attribute::NoUnwind) .Case("optnone", Attribute::OptimizeNone) Index: lib/Transforms/Utils/CodeExtractor.cpp =================================================================== --- lib/Transforms/Utils/CodeExtractor.cpp +++ lib/Transforms/Utils/CodeExtractor.cpp @@ -695,6 +695,7 @@ case Attribute::StackProtectStrong: case Attribute::StrictFP: case Attribute::UWTable: + case Attribute::NoCfCheck: break; } Index: test/CodeGen/X86/indirect-branch-tracking.ll =================================================================== --- test/CodeGen/X86/indirect-branch-tracking.ll +++ test/CodeGen/X86/indirect-branch-tracking.ll @@ -1,10 +1,11 @@ -; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+ibt -x86-indirect-branch-tracking < %s | FileCheck %s --check-prefix=ALL --check-prefix=X86_64 -; RUN: llc -mtriple=i386-unknown-unknown -mattr=+ibt -x86-indirect-branch-tracking < %s | FileCheck %s --check-prefix=ALL --check-prefix=X86 +; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+ibt < %s | FileCheck %s --check-prefix=ALL --check-prefix=X86_64 +; RUN: llc -mtriple=i386-unknown-unknown -mattr=+ibt < %s | FileCheck %s --check-prefix=ALL --check-prefix=X86 +; RUN: llc -mtriple i386-windows-gnu -mattr=+ibt -exception-model sjlj < %s | FileCheck %s --check-prefix=SJLJ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Test1 ;; ----- -;; Checks ENDBR insertion in case of indirect branch IR instruction. +;; Checks ENDBR insertion in case of switch case statement. ;; Also since the function is not internal, make sure that endbr32/64 was ;; added at the beginning of the function. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -34,7 +35,8 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Test2 ;; ----- -;; Checks ENDBR insertion in case of switch case statement. +;; Checks NOTRACK insertion in case of switch case statement. +;; Check that there is no ENDBR insertion in the following case statements. ;; Also since the function is not internal, ENDBR instruction should be ;; added to its first basic block. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -43,25 +45,9 @@ ; ALL-LABEL: test2 ; X86_64: endbr64 ; X86: endbr32 -; ALL: jmp{{q|l}} * -; ALL: .LBB1_2: -; X86_64-NEXT: endbr64 -; X86-NEXT: endbr32 -; ALL: .LBB1_7: +; ALL: notrack jmp{{q|l}} * ; X86_64-NOT: endbr64 ; X86-NOT: endbr32 -; ALL: .LBB1_3: -; X86_64-NEXT: endbr64 -; X86-NEXT: endbr32 -; ALL: .LBB1_4: -; X86_64-NEXT: endbr64 -; X86-NEXT: endbr32 -; ALL: .LBB1_5: -; X86_64-NEXT: endbr64 -; X86-NEXT: endbr32 -; ALL: .LBB1_6: -; X86_64-NEXT: endbr64 -; X86-NEXT: endbr32 entry: %retval = alloca i32, align 4 %a.addr = alloca i32, align 4 @@ -198,3 +184,38 @@ ; X86: endbr32 ret i32 1 } + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Test8 +;; ----- +;; Checks that NO TRACK prefix is not added for indirect jumps to a jump- +;; table that was created for SJLJ dispatch. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +declare void @_Z20function_that_throwsv() +declare i32 @__gxx_personality_sj0(...) +declare i8* @__cxa_begin_catch(i8*) +declare void @__cxa_end_catch() + +define void @test8() personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) { +;SJLJ-LABEL: test8 +;SJLJ-NOT: ds +entry: + invoke void @_Z20function_that_throwsv() + to label %try.cont unwind label %lpad + +lpad: + %0 = landingpad { i8*, i32 } + catch i8* null + %1 = extractvalue { i8*, i32 } %0, 0 + %2 = tail call i8* @__cxa_begin_catch(i8* %1) + tail call void @__cxa_end_catch() + br label %try.cont + +try.cont: + ret void +} + +!llvm.module.flags = !{!0} + +!0 = !{i32 4, !"cf-protection-branch", i32 1} \ No newline at end of file Index: test/CodeGen/X86/nocf_check.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/nocf_check.ll @@ -0,0 +1,46 @@ +; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+ibt -x86-indirect-branch-tracking < %s | FileCheck %s + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; This test verify the handling of ''nocf_check'' attribute by the backend. ;; +;; The file was generated using the following C code: ;; +;; ;; +;; void __attribute__((nocf_check)) NoCfCheckFunc(void) {} ;; +;; ;; +;; typedef void(*FuncPointer)(void); ;; +;; void NoCfCheckCall(FuncPointer f) { ;; +;; __attribute__((nocf_check)) FuncPointer p = f; ;; +;; (*p)(); ;; +;; } ;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; Make sure that a function with ''nocf_check'' attribute is not instrumented +; with endbr instruction at the beginning. +define void @NoCfCheckFunc() #0 { +; CHECK-LABEL: NoCfCheckFunc +; CHECK-NOT: endbr64 +; CHECK: retq +entry: + ret void +} + +; Make sure that notrack prefix is added before a call with ''nocf_check'' attribute. +define void @NoCfCheckCall(void ()* %f) { +; CHECK-LABEL: NoCfCheckCall +; CHECK: notrack call +entry: + %f.addr = alloca void ()*, align 4 + %p = alloca void ()*, align 4 + store void ()* %f, void ()** %f.addr, align 4 + %0 = load void ()*, void ()** %f.addr, align 4 + store void ()* %0, void ()** %p, align 4 + %1 = load void ()*, void ()** %p, align 4 + call void %1() #1 + ret void +} + +attributes #0 = { noinline nocf_check nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nocf_check } + +!llvm.module.flags = !{!0} + +!0 = !{i32 4, !"cf-protection-branch", i32 1} Index: test/MC/X86/x86-32-coverage.s =================================================================== --- test/MC/X86/x86-32-coverage.s +++ test/MC/X86/x86-32-coverage.s @@ -1655,10 +1655,18 @@ // CHECK: encoding: [0xff,0xd1] call *%ecx +// CHECK: notrack calll *%ecx +// CHECK: encoding: [0x3e,0xff,0xd1] + notrack call *%ecx + // CHECK: calll *3735928559(%ebx,%ecx,8) // CHECK: encoding: [0xff,0x94,0xcb,0xef,0xbe,0xad,0xde] call *0xdeadbeef(%ebx,%ecx,8) +// CHECK: notrack calll *3735928559(%ebx,%ecx,8) +// CHECK: encoding: [0x3e,0xff,0x94,0xcb,0xef,0xbe,0xad,0xde] + notrack call *0xdeadbeef(%ebx,%ecx,8) + // CHECK: calll *3135175374 // CHECK: encoding: [0xff,0x15,0xce,0xfa,0xde,0xba] call *0xbadeface @@ -1687,6 +1695,10 @@ // CHECK: encoding: [0xff,0xa4,0xcb,0xef,0xbe,0xad,0xde] jmp *0xdeadbeef(%ebx,%ecx,8) +// CHECK: notrack jmpl *3735928559(%ebx,%ecx,8) +// CHECK: encoding: [0x3e,0xff,0xa4,0xcb,0xef,0xbe,0xad,0xde] + notrack jmp *0xdeadbeef(%ebx,%ecx,8) + // CHECK: jmpl *3135175374 // CHECK: encoding: [0xff,0x25,0xce,0xfa,0xde,0xba] jmp *0xbadeface Index: utils/TableGen/X86FoldTablesEmitter.cpp =================================================================== --- utils/TableGen/X86FoldTablesEmitter.cpp +++ utils/TableGen/X86FoldTablesEmitter.cpp @@ -350,6 +350,8 @@ MemRec->getValueAsBit("hasREX_WPrefix") || RegRec->getValueAsBit("hasLockPrefix") != MemRec->getValueAsBit("hasLockPrefix") || + RegRec->getValueAsBit("hasNoTrackPrefix") != + MemRec->getValueAsBit("hasNoTrackPrefix") || !equalBitsInits(RegRec->getValueAsBitsInit("EVEX_LL"), MemRec->getValueAsBitsInit("EVEX_LL")) || !equalBitsInits(RegRec->getValueAsBitsInit("VEX_WPrefix"),