diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h --- a/llvm/lib/Target/X86/X86.h +++ b/llvm/lib/Target/X86/X86.h @@ -141,7 +141,6 @@ X86RegisterBankInfo &); FunctionPass *createX86LoadValueInjectionLoadHardeningPass(); -FunctionPass *createX86LoadValueInjectionLoadHardeningUnoptimizedPass(); FunctionPass *createX86LoadValueInjectionRetHardeningPass(); FunctionPass *createX86SpeculativeLoadHardeningPass(); FunctionPass *createX86SpeculativeExecutionSideEffectSuppression(); @@ -161,7 +160,6 @@ void initializeX86ExpandPseudoPass(PassRegistry &); void initializeX86FixupSetCCPassPass(PassRegistry &); void initializeX86FlagsCopyLoweringPassPass(PassRegistry &); -void initializeX86LoadValueInjectionLoadHardeningUnoptimizedPassPass(PassRegistry &); void initializeX86LoadValueInjectionLoadHardeningPassPass(PassRegistry &); void initializeX86LoadValueInjectionRetHardeningPassPass(PassRegistry &); void initializeX86OptimizeLEAPassPass(PassRegistry &); diff --git a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp --- a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp +++ b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp @@ -822,79 +822,3 @@ FunctionPass *llvm::createX86LoadValueInjectionLoadHardeningPass() { return new X86LoadValueInjectionLoadHardeningPass(); } - -namespace { - -/// The `X86LoadValueInjectionLoadHardeningPass` above depends on expensive -/// analysis passes that add complexity to the pipeline. This complexity -/// can cause noticable overhead when no optimizations are enabled, i.e., -O0. -/// The purpose of `X86LoadValueInjectionLoadHardeningUnoptimizedPass` is to -/// provide the same security as the optimized pass, but without adding -/// unnecessary complexity to the LLVM pipeline. -/// -/// The behavior of this pass is simply to insert an LFENCE after every load -/// instruction. -class X86LoadValueInjectionLoadHardeningUnoptimizedPass - : public MachineFunctionPass { -public: - X86LoadValueInjectionLoadHardeningUnoptimizedPass() - : MachineFunctionPass(ID) {} - - StringRef getPassName() const override { - return "X86 Load Value Injection (LVI) Load Hardening (Unoptimized)"; - } - bool runOnMachineFunction(MachineFunction &MF) override; - static char ID; -}; - -} // end anonymous namespace - -char X86LoadValueInjectionLoadHardeningUnoptimizedPass::ID = 0; - -bool X86LoadValueInjectionLoadHardeningUnoptimizedPass::runOnMachineFunction( - MachineFunction &MF) { - LLVM_DEBUG(dbgs() << "***** " << getPassName() << " : " << MF.getName() - << " *****\n"); - const X86Subtarget *STI = &MF.getSubtarget(); - if (!STI->useLVILoadHardening()) - return false; - - // FIXME: support 32-bit - if (!STI->is64Bit()) - report_fatal_error("LVI load hardening is only supported on 64-bit", false); - - // Don't skip functions with the "optnone" attr but participate in opt-bisect. - const Function &F = MF.getFunction(); - if (!F.hasOptNone() && skipFunction(F)) - return false; - - bool Modified = false; - ++NumFunctionsConsidered; - - const TargetInstrInfo *TII = STI->getInstrInfo(); - for (auto &MBB : MF) { - for (auto &MI : MBB) { - if (!MI.mayLoad() || MI.getOpcode() == X86::LFENCE || - MI.getOpcode() == X86::MFENCE) - continue; - - MachineBasicBlock::iterator InsertionPt = - MI.getNextNode() ? MI.getNextNode() : MBB.end(); - BuildMI(MBB, InsertionPt, DebugLoc(), TII->get(X86::LFENCE)); - ++NumFences; - Modified = true; - } - } - - if (Modified) - ++NumFunctionsMitigated; - - return Modified; -} - -INITIALIZE_PASS(X86LoadValueInjectionLoadHardeningUnoptimizedPass, PASS_KEY, - "X86 LVI load hardening", false, false) - -FunctionPass *llvm::createX86LoadValueInjectionLoadHardeningUnoptimizedPass() { - return new X86LoadValueInjectionLoadHardeningUnoptimizedPass(); -} diff --git a/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp b/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp --- a/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp +++ b/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp @@ -22,6 +22,7 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/Pass.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; #define DEBUG_TYPE "x86-seses" @@ -86,13 +87,19 @@ bool X86SpeculativeExecutionSideEffectSuppression::runOnMachineFunction( MachineFunction &MF) { - if (!EnableSpeculativeExecutionSideEffectSuppression) + + const auto &OptLevel = MF.getTarget().getOptLevel(); + const X86Subtarget &Subtarget = MF.getSubtarget(); + + // Check whether SESES needs to run as the fallback for LVI at O0 or if the + // user explicitly passed the SESES flag. + if (!EnableSpeculativeExecutionSideEffectSuppression && + !(Subtarget.useLVILoadHardening() && OptLevel == CodeGenOpt::None)) return false; LLVM_DEBUG(dbgs() << "********** " << getPassName() << " : " << MF.getName() << " **********\n"); bool Modified = false; - const X86Subtarget &Subtarget = MF.getSubtarget(); const X86InstrInfo *TII = Subtarget.getInstrInfo(); for (MachineBasicBlock &MBB : MF) { MachineInstr *FirstTerminator = nullptr; diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -491,8 +491,6 @@ addPass(createX86FloatingPointStackifierPass()); if (getOptLevel() != CodeGenOpt::None) addPass(createX86LoadValueInjectionLoadHardeningPass()); - else - addPass(createX86LoadValueInjectionLoadHardeningUnoptimizedPass()); } void X86PassConfig::addPreSched2() { addPass(createX86ExpandPseudoPass()); } diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll --- a/llvm/test/CodeGen/X86/O0-pipeline.ll +++ b/llvm/test/CodeGen/X86/O0-pipeline.ll @@ -46,7 +46,6 @@ ; CHECK-NEXT: Fast Register Allocator ; CHECK-NEXT: Bundle Machine CFG Edges ; CHECK-NEXT: X86 FP Stackifier -; CHECK-NEXT: X86 Load Value Injection (LVI) Load Hardening (Unoptimized) ; CHECK-NEXT: Fixup Statepoint Caller Saved ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Machine Optimization Remark Emitter diff --git a/llvm/test/CodeGen/X86/lvi-hardening-loads.ll b/llvm/test/CodeGen/X86/lvi-hardening-loads.ll --- a/llvm/test/CodeGen/X86/lvi-hardening-loads.ll +++ b/llvm/test/CodeGen/X86/lvi-hardening-loads.ll @@ -26,10 +26,15 @@ ; X64-NEXT: jmp .LBB0_1 ; X64-NOOPT: # %bb.0: # %entry +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movq %rdi, -{{[0-9]+}}(%rsp) +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movl %esi, -{{[0-9]+}}(%rsp) +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movl $0, -{{[0-9]+}}(%rsp) ; X64-NOOPT-NEXT: lfence +; X64-NOOPT-NEXT: lfence +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movl $0, -{{[0-9]+}}(%rsp) for.cond: ; preds = %for.inc, %entry @@ -48,6 +53,7 @@ ; X64-NOOPT: .LBB0_1: # %for.cond ; X64-NOOPT-NEXT: # =>This Inner Loop Header: Depth=1 +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax ; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: cmpl -{{[0-9]+}}(%rsp), %eax @@ -73,12 +79,13 @@ ; X64-NOOPT: # %bb.2: # %for.body ; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1 -; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax ; X64-NOOPT-NEXT: lfence +; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax ; X64-NOOPT-NEXT: cltd ; X64-NOOPT-NEXT: movl $2, %ecx ; X64-NOOPT-NEXT: idivl %ecx ; X64-NOOPT-NEXT: cmpl $0, %edx +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: jne .LBB0_4 if.then: ; preds = %for.body @@ -105,6 +112,7 @@ ; X64-NOOPT: # %bb.3: # %if.then ; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1 +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movq -{{[0-9]+}}(%rsp), %rax ; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movslq -{{[0-9]+}}(%rsp), %rcx @@ -126,10 +134,12 @@ ; X64-NOOPT: .LBB0_5: # %for.inc ; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1 -; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax ; X64-NOOPT-NEXT: lfence +; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax ; X64-NOOPT-NEXT: addl $1, %eax +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: movl %eax, -{{[0-9]+}}(%rsp) +; X64-NOOPT-NEXT: lfence ; X64-NOOPT-NEXT: jmp .LBB0_1 for.end: ; preds = %for.cond