Changeset View
Changeset View
Standalone View
Standalone View
llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp
| Show First 20 Lines • Show All 1,584 Lines • ▼ Show 20 Lines | static bool isDataInvariantLoad(MachineInstr &MI) { | ||||
| case X86::MOVZX32rm8: | case X86::MOVZX32rm8: | ||||
| case X86::MOVZX32rm8_NOREX: | case X86::MOVZX32rm8_NOREX: | ||||
| case X86::MOVZX64rm16: | case X86::MOVZX64rm16: | ||||
| case X86::MOVZX64rm8: | case X86::MOVZX64rm8: | ||||
| return true; | return true; | ||||
| } | } | ||||
| } | } | ||||
| // Returns true if the MI has EFLAGS as a register def operand and it's live, | |||||
| // otherwise it returns false | |||||
| static bool isEFLAGSDefLive(const MachineInstr &MI) { | |||||
| if (const MachineOperand *DefOp = MI.findRegisterDefOperand(X86::EFLAGS)) { | |||||
| return !DefOp->isDead(); | |||||
| } | |||||
| return false; | |||||
| } | |||||
| static bool isEFLAGSLive(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, | static bool isEFLAGSLive(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, | ||||
| const TargetRegisterInfo &TRI) { | const TargetRegisterInfo &TRI) { | ||||
| // Check if EFLAGS are alive by seeing if there is a def of them or they | // Check if EFLAGS are alive by seeing if there is a def of them or they | ||||
| // live-in, and then seeing if that def is in turn used. | // live-in, and then seeing if that def is in turn used. | ||||
| for (MachineInstr &MI : llvm::reverse(llvm::make_range(MBB.begin(), I))) { | for (MachineInstr &MI : llvm::reverse(llvm::make_range(MBB.begin(), I))) { | ||||
| if (MachineOperand *DefOp = MI.findRegisterDefOperand(X86::EFLAGS)) { | if (MachineOperand *DefOp = MI.findRegisterDefOperand(X86::EFLAGS)) { | ||||
| // If the def is dead, then EFLAGS is not live. | // If the def is dead, then EFLAGS is not live. | ||||
| if (DefOp->isDead()) | if (DefOp->isDead()) | ||||
| ▲ Show 20 Lines • Show All 140 Lines • ▼ Show 20 Lines | if (HardenLoads) | ||||
| continue; | continue; | ||||
| // If post-load hardening is enabled, this load is compatible with | // If post-load hardening is enabled, this load is compatible with | ||||
| // post-load hardening, and we aren't already going to harden one of the | // post-load hardening, and we aren't already going to harden one of the | ||||
| // address registers, queue it up to be hardened post-load. Notably, | // address registers, queue it up to be hardened post-load. Notably, | ||||
| // even once hardened this won't introduce a useful dependency that | // even once hardened this won't introduce a useful dependency that | ||||
| // could prune out subsequent loads. | // could prune out subsequent loads. | ||||
| if (EnablePostLoadHardening && isDataInvariantLoad(MI) && | if (EnablePostLoadHardening && isDataInvariantLoad(MI) && | ||||
| MI.getDesc().getNumDefs() == 1 && MI.getOperand(0).isReg() && | !isEFLAGSDefLive(MI) && MI.getDesc().getNumDefs() == 1 && | ||||
| MI.getOperand(0).isReg() && | |||||
| canHardenRegister(MI.getOperand(0).getReg()) && | canHardenRegister(MI.getOperand(0).getReg()) && | ||||
| !HardenedAddrRegs.count(BaseReg) && | !HardenedAddrRegs.count(BaseReg) && | ||||
| !HardenedAddrRegs.count(IndexReg)) { | !HardenedAddrRegs.count(IndexReg)) { | ||||
| HardenPostLoad.insert(&MI); | HardenPostLoad.insert(&MI); | ||||
| HardenedAddrRegs.insert(MI.getOperand(0).getReg()); | HardenedAddrRegs.insert(MI.getOperand(0).getReg()); | ||||
| continue; | continue; | ||||
| } | } | ||||
| Show All 37 Lines | for (MachineInstr &MI : MBB) { | ||||
| continue; | continue; | ||||
| } | } | ||||
| // Test if this instruction is one of our post load instructions (and | // Test if this instruction is one of our post load instructions (and | ||||
| // remove it from the set if so). | // remove it from the set if so). | ||||
| if (HardenPostLoad.erase(&MI)) { | if (HardenPostLoad.erase(&MI)) { | ||||
| assert(!MI.isCall() && "Must not try to post-load harden a call!"); | assert(!MI.isCall() && "Must not try to post-load harden a call!"); | ||||
| // If this is a data-invariant load, we want to try and sink any | // If this is a data-invariant load and there is no EFLAGS | ||||
| // hardening as far as possible. | // interference, we want to try and sink any hardening as far as | ||||
| if (isDataInvariantLoad(MI)) { | // possible. | ||||
| if (isDataInvariantLoad(MI) && !isEFLAGSDefLive(MI)) { | |||||
| // Sink the instruction we'll need to harden as far as we can down | // Sink the instruction we'll need to harden as far as we can down | ||||
| // the graph. | // the graph. | ||||
| MachineInstr *SunkMI = sinkPostLoadHardenedInst(MI, HardenPostLoad); | MachineInstr *SunkMI = sinkPostLoadHardenedInst(MI, HardenPostLoad); | ||||
| // If we managed to sink this instruction, update everything so we | // If we managed to sink this instruction, update everything so we | ||||
| // harden that instruction when we reach it in the instruction | // harden that instruction when we reach it in the instruction | ||||
| // sequence. | // sequence. | ||||
| if (SunkMI != &MI) { | if (SunkMI != &MI) { | ||||
| ▲ Show 20 Lines • Show All 335 Lines • ▼ Show 20 Lines | void X86SpeculativeLoadHardeningPass::hardenLoadAddr( | ||||
| if (FlagsReg) | if (FlagsReg) | ||||
| restoreEFLAGS(MBB, InsertPt, Loc, FlagsReg); | restoreEFLAGS(MBB, InsertPt, Loc, FlagsReg); | ||||
| } | } | ||||
| MachineInstr *X86SpeculativeLoadHardeningPass::sinkPostLoadHardenedInst( | MachineInstr *X86SpeculativeLoadHardeningPass::sinkPostLoadHardenedInst( | ||||
| MachineInstr &InitialMI, SmallPtrSetImpl<MachineInstr *> &HardenedInstrs) { | MachineInstr &InitialMI, SmallPtrSetImpl<MachineInstr *> &HardenedInstrs) { | ||||
| assert(isDataInvariantLoad(InitialMI) && | assert(isDataInvariantLoad(InitialMI) && | ||||
| "Cannot get here with a non-invariant load!"); | "Cannot get here with a non-invariant load!"); | ||||
| assert(!isEFLAGSDefLive(InitialMI) && | |||||
| "Cannot get here with a data invariant load " | |||||
| "that interferes with EFLAGS!"); | |||||
| // See if we can sink hardening the loaded value. | // See if we can sink hardening the loaded value. | ||||
| auto SinkCheckToSingleUse = | auto SinkCheckToSingleUse = | ||||
| [&](MachineInstr &MI) -> Optional<MachineInstr *> { | [&](MachineInstr &MI) -> Optional<MachineInstr *> { | ||||
| Register DefReg = MI.getOperand(0).getReg(); | Register DefReg = MI.getOperand(0).getReg(); | ||||
| // We need to find a single use which we can sink the check. We can | // We need to find a single use which we can sink the check. We can | ||||
| // primarily do this because many uses may already end up checked on their | // primarily do this because many uses may already end up checked on their | ||||
| // own. | // own. | ||||
| MachineInstr *SingleUseMI = nullptr; | MachineInstr *SingleUseMI = nullptr; | ||||
| for (MachineInstr &UseMI : MRI->use_instructions(DefReg)) { | for (MachineInstr &UseMI : MRI->use_instructions(DefReg)) { | ||||
| // If we're already going to harden this use, it is data invariant and | // If we're already going to harden this use, it is data invariant, it | ||||
| // within our block. | // does not interfere with EFLAGS, and within our block. | ||||
| if (HardenedInstrs.count(&UseMI)) { | if (HardenedInstrs.count(&UseMI)) { | ||||
| if (!isDataInvariantLoad(UseMI)) { | if (!isDataInvariantLoad(UseMI) || isEFLAGSDefLive(UseMI)) { | ||||
| // If we've already decided to harden a non-load, we must have sunk | // If we've already decided to harden a non-load, we must have sunk | ||||
| // some other post-load hardened instruction to it and it must itself | // some other post-load hardened instruction to it and it must itself | ||||
| // be data-invariant. | // be data-invariant. | ||||
| assert(isDataInvariant(UseMI) && | assert(isDataInvariant(UseMI) && | ||||
| "Data variant instruction being hardened!"); | "Data variant instruction being hardened!"); | ||||
| continue; | continue; | ||||
| } | } | ||||
| Show All 19 Lines | for (MachineInstr &UseMI : MRI->use_instructions(DefReg)) { | ||||
| } | } | ||||
| if (SingleUseMI) | if (SingleUseMI) | ||||
| // We already have a single use, this would make two. Bail. | // We already have a single use, this would make two. Bail. | ||||
| return {}; | return {}; | ||||
| // If this single use isn't data invariant, isn't in this block, or has | // If this single use isn't data invariant, isn't in this block, or has | ||||
| // interfering EFLAGS, we can't sink the hardening to it. | // interfering EFLAGS, we can't sink the hardening to it. | ||||
| if (!isDataInvariant(UseMI) || UseMI.getParent() != MI.getParent()) | if (!isDataInvariant(UseMI) || UseMI.getParent() != MI.getParent() || | ||||
| isEFLAGSDefLive(UseMI)) { | |||||
MaskRay: Delete excess parentheses. | |||||
zbridAuthorUnsubmitted Hm, I'm not sure which parentheses you mean. On my scan, I don't see any extras. zbrid: Hm, I'm not sure which parentheses you mean. On my scan, I don't see any extras. | |||||
| return {}; | return {}; | ||||
| } | |||||
| // If this instruction defines multiple registers bail as we won't harden | // If this instruction defines multiple registers bail as we won't harden | ||||
| // all of them. | // all of them. | ||||
| if (UseMI.getDesc().getNumDefs() > 1) | if (UseMI.getDesc().getNumDefs() > 1) | ||||
| return {}; | return {}; | ||||
| // If this register isn't a virtual register we can't walk uses of sanely, | // If this register isn't a virtual register we can't walk uses of sanely, | ||||
| // just bail. Also check that its register class is one of the ones we | // just bail. Also check that its register class is one of the ones we | ||||
| ▲ Show 20 Lines • Show All 437 Lines • Show Last 20 Lines | |||||
Delete excess parentheses.