diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h --- a/llvm/lib/Target/X86/X86.h +++ b/llvm/lib/Target/X86/X86.h @@ -141,7 +141,6 @@ X86RegisterBankInfo &); FunctionPass *createX86LoadValueInjectionLoadHardeningPass(); -FunctionPass *createX86LoadValueInjectionLoadHardeningUnoptimizedPass(); FunctionPass *createX86LoadValueInjectionRetHardeningPass(); FunctionPass *createX86SpeculativeLoadHardeningPass(); FunctionPass *createX86SpeculativeExecutionSideEffectSuppression(); @@ -161,7 +160,6 @@ void initializeX86ExpandPseudoPass(PassRegistry &); void initializeX86FixupSetCCPassPass(PassRegistry &); void initializeX86FlagsCopyLoweringPassPass(PassRegistry &); -void initializeX86LoadValueInjectionLoadHardeningUnoptimizedPassPass(PassRegistry &); void initializeX86LoadValueInjectionLoadHardeningPassPass(PassRegistry &); void initializeX86LoadValueInjectionRetHardeningPassPass(PassRegistry &); void initializeX86OptimizeLEAPassPass(PassRegistry &); diff --git a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp --- a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp +++ b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp @@ -822,79 +822,3 @@ FunctionPass *llvm::createX86LoadValueInjectionLoadHardeningPass() { return new X86LoadValueInjectionLoadHardeningPass(); } - -namespace { - -/// The `X86LoadValueInjectionLoadHardeningPass` above depends on expensive -/// analysis passes that add complexity to the pipeline. This complexity -/// can cause noticable overhead when no optimizations are enabled, i.e., -O0. -/// The purpose of `X86LoadValueInjectionLoadHardeningUnoptimizedPass` is to -/// provide the same security as the optimized pass, but without adding -/// unnecessary complexity to the LLVM pipeline. -/// -/// The behavior of this pass is simply to insert an LFENCE after every load -/// instruction. -class X86LoadValueInjectionLoadHardeningUnoptimizedPass - : public MachineFunctionPass { -public: - X86LoadValueInjectionLoadHardeningUnoptimizedPass() - : MachineFunctionPass(ID) {} - - StringRef getPassName() const override { - return "X86 Load Value Injection (LVI) Load Hardening (Unoptimized)"; - } - bool runOnMachineFunction(MachineFunction &MF) override; - static char ID; -}; - -} // end anonymous namespace - -char X86LoadValueInjectionLoadHardeningUnoptimizedPass::ID = 0; - -bool X86LoadValueInjectionLoadHardeningUnoptimizedPass::runOnMachineFunction( - MachineFunction &MF) { - LLVM_DEBUG(dbgs() << "***** " << getPassName() << " : " << MF.getName() - << " *****\n"); - const X86Subtarget *STI = &MF.getSubtarget(); - if (!STI->useLVILoadHardening()) - return false; - - // FIXME: support 32-bit - if (!STI->is64Bit()) - report_fatal_error("LVI load hardening is only supported on 64-bit", false); - - // Don't skip functions with the "optnone" attr but participate in opt-bisect. - const Function &F = MF.getFunction(); - if (!F.hasOptNone() && skipFunction(F)) - return false; - - bool Modified = false; - ++NumFunctionsConsidered; - - const TargetInstrInfo *TII = STI->getInstrInfo(); - for (auto &MBB : MF) { - for (auto &MI : MBB) { - if (!MI.mayLoad() || MI.getOpcode() == X86::LFENCE || - MI.getOpcode() == X86::MFENCE) - continue; - - MachineBasicBlock::iterator InsertionPt = - MI.getNextNode() ? MI.getNextNode() : MBB.end(); - BuildMI(MBB, InsertionPt, DebugLoc(), TII->get(X86::LFENCE)); - ++NumFences; - Modified = true; - } - } - - if (Modified) - ++NumFunctionsMitigated; - - return Modified; -} - -INITIALIZE_PASS(X86LoadValueInjectionLoadHardeningUnoptimizedPass, PASS_KEY, - "X86 LVI load hardening", false, false) - -FunctionPass *llvm::createX86LoadValueInjectionLoadHardeningUnoptimizedPass() { - return new X86LoadValueInjectionLoadHardeningUnoptimizedPass(); -} diff --git a/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp b/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp --- a/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp +++ b/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp @@ -22,6 +22,7 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/Pass.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; #define DEBUG_TYPE "x86-seses" @@ -86,27 +87,42 @@ bool X86SpeculativeExecutionSideEffectSuppression::runOnMachineFunction( MachineFunction &MF) { - if (!EnableSpeculativeExecutionSideEffectSuppression) + + const auto &OptLevel = MF.getTarget().getOptLevel(); + const X86Subtarget &Subtarget = MF.getSubtarget(); + + // Check whether SESES needs to run as the fallback for LVI at O0 or if the + // user explicitly passed the SESES flag. + if (!EnableSpeculativeExecutionSideEffectSuppression && + !(Subtarget.useLVILoadHardening() && OptLevel == CodeGenOpt::None)) return false; LLVM_DEBUG(dbgs() << "********** " << getPassName() << " : " << MF.getName() << " **********\n"); bool Modified = false; - const X86Subtarget &Subtarget = MF.getSubtarget(); const X86InstrInfo *TII = Subtarget.getInstrInfo(); for (MachineBasicBlock &MBB : MF) { MachineInstr *FirstTerminator = nullptr; - + // Keep track of whether the previous instruction was an LFENCE to avoid + // adding redundant LFENCEs. + bool PrevInstIsLFENCE = false; for (auto &MI : MBB) { + + if (MI.getOpcode() == X86::LFENCE) { + PrevInstIsLFENCE = true; + continue; + } // We want to put an LFENCE before any instruction that // may load or store. This LFENCE is intended to avoid leaking any secret // data due to a given load or store. This results in closing the cache // and memory timing side channels. We will treat terminators that load // or store separately. if (MI.mayLoadOrStore() && !MI.isTerminator()) { - BuildMI(MBB, MI, DebugLoc(), TII->get(X86::LFENCE)); - NumLFENCEsInserted++; - Modified = true; + if (!PrevInstIsLFENCE) { + BuildMI(MBB, MI, DebugLoc(), TII->get(X86::LFENCE)); + NumLFENCEsInserted++; + Modified = true; + } if (OneLFENCEPerBasicBlock) break; } @@ -128,19 +144,25 @@ // Look for branch instructions that will require an LFENCE to be put // before this basic block's terminators. - if (!MI.isBranch() || OmitBranchLFENCEs) + if (!MI.isBranch() || OmitBranchLFENCEs) { // This isn't a branch or we're not putting LFENCEs before branches. + PrevInstIsLFENCE = false; continue; + } - if (OnlyLFENCENonConst && hasConstantAddressingMode(MI)) + if (OnlyLFENCENonConst && hasConstantAddressingMode(MI)) { // This is a branch, but it only has constant addressing mode and we're // not adding LFENCEs before such branches. + PrevInstIsLFENCE = false; continue; + } // This branch requires adding an LFENCE. - BuildMI(MBB, FirstTerminator, DebugLoc(), TII->get(X86::LFENCE)); - NumLFENCEsInserted++; - Modified = true; + if (!PrevInstIsLFENCE) { + BuildMI(MBB, FirstTerminator, DebugLoc(), TII->get(X86::LFENCE)); + NumLFENCEsInserted++; + Modified = true; + } break; } } diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -489,10 +489,12 @@ void X86PassConfig::addPostRegAlloc() { addPass(createX86FloatingPointStackifierPass()); + // When -O0 is enabled, the Load Value Injection Hardening pass will fall back + // to using the Speculative Execution Side Effect Suppression pass for + // mitigation. This is to prevent slow downs due to + // analyses needed by the LVIHardening pass when compiling at -O0. if (getOptLevel() != CodeGenOpt::None) addPass(createX86LoadValueInjectionLoadHardeningPass()); - else - addPass(createX86LoadValueInjectionLoadHardeningUnoptimizedPass()); } void X86PassConfig::addPreSched2() { addPass(createX86ExpandPseudoPass()); } diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll --- a/llvm/test/CodeGen/X86/O0-pipeline.ll +++ b/llvm/test/CodeGen/X86/O0-pipeline.ll @@ -46,7 +46,6 @@ ; CHECK-NEXT: Fast Register Allocator ; CHECK-NEXT: Bundle Machine CFG Edges ; CHECK-NEXT: X86 FP Stackifier -; CHECK-NEXT: X86 Load Value Injection (LVI) Load Hardening (Unoptimized) ; CHECK-NEXT: Fixup Statepoint Caller Saved ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Machine Optimization Remark Emitter diff --git a/llvm/test/CodeGen/X86/lvi-hardening-loads.ll b/llvm/test/CodeGen/X86/lvi-hardening-loads.ll --- a/llvm/test/CodeGen/X86/lvi-hardening-loads.ll +++ b/llvm/test/CodeGen/X86/lvi-hardening-loads.ll @@ -26,8 +26,11 @@ ; X64-NEXT: jmp .LBB0_1 ; X64-NOOPT: # %bb.0: # %entry +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movq %rdi, -{{[0-9]+}}(%rsp) +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movl %esi, -{{[0-9]+}}(%rsp) +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movl $0, -{{[0-9]+}}(%rsp) ; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movl $0, -{{[0-9]+}}(%rsp) @@ -48,6 +51,7 @@ ; X64-NOOPT: .LBB0_1: # %for.cond ; X64-NOOPT-NEXT: # =>This Inner Loop Header: Depth=1 +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax ; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: cmpl -{{[0-9]+}}(%rsp), %eax @@ -73,12 +77,13 @@ ; X64-NOOPT: # %bb.2: # %for.body ; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1 -; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax ; X64-NOOPT-NEXT: lfence +; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax ; X64-NOOPT-NEXT: cltd ; X64-NOOPT-NEXT: movl $2, %ecx ; X64-NOOPT-NEXT: idivl %ecx ; X64-NOOPT-NEXT: cmpl $0, %edx +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: jne .LBB0_4 if.then: ; preds = %for.body @@ -105,6 +110,7 @@ ; X64-NOOPT: # %bb.3: # %if.then ; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1 +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movq -{{[0-9]+}}(%rsp), %rax ; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movslq -{{[0-9]+}}(%rsp), %rcx @@ -126,10 +132,12 @@ ; X64-NOOPT: .LBB0_5: # %for.inc ; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1 -; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax ; X64-NOOPT-NEXT: lfence +; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax ; X64-NOOPT-NEXT: addl $1, %eax +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movl %eax, -{{[0-9]+}}(%rsp) +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: jmp .LBB0_1 for.end: ; preds = %for.cond