diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h
--- a/llvm/lib/Target/X86/X86.h
+++ b/llvm/lib/Target/X86/X86.h
@@ -141,7 +141,6 @@
X86RegisterBankInfo &);
FunctionPass *createX86LoadValueInjectionLoadHardeningPass();
-FunctionPass *createX86LoadValueInjectionLoadHardeningUnoptimizedPass();
FunctionPass *createX86LoadValueInjectionRetHardeningPass();
FunctionPass *createX86SpeculativeLoadHardeningPass();
FunctionPass *createX86SpeculativeExecutionSideEffectSuppression();
@@ -161,7 +160,6 @@
void initializeX86ExpandPseudoPass(PassRegistry &);
void initializeX86FixupSetCCPassPass(PassRegistry &);
void initializeX86FlagsCopyLoweringPassPass(PassRegistry &);
-void initializeX86LoadValueInjectionLoadHardeningUnoptimizedPassPass(PassRegistry &);
void initializeX86LoadValueInjectionLoadHardeningPassPass(PassRegistry &);
void initializeX86LoadValueInjectionRetHardeningPassPass(PassRegistry &);
void initializeX86OptimizeLEAPassPass(PassRegistry &);
diff --git a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
--- a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
+++ b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
@@ -822,79 +822,3 @@
FunctionPass *llvm::createX86LoadValueInjectionLoadHardeningPass() {
return new X86LoadValueInjectionLoadHardeningPass();
}
-
-namespace {
-
-/// The `X86LoadValueInjectionLoadHardeningPass` above depends on expensive
-/// analysis passes that add complexity to the pipeline. This complexity
-/// can cause noticable overhead when no optimizations are enabled, i.e., -O0.
-/// The purpose of `X86LoadValueInjectionLoadHardeningUnoptimizedPass` is to
-/// provide the same security as the optimized pass, but without adding
-/// unnecessary complexity to the LLVM pipeline.
-///
-/// The behavior of this pass is simply to insert an LFENCE after every load
-/// instruction.
-class X86LoadValueInjectionLoadHardeningUnoptimizedPass
- : public MachineFunctionPass {
-public:
- X86LoadValueInjectionLoadHardeningUnoptimizedPass()
- : MachineFunctionPass(ID) {}
-
- StringRef getPassName() const override {
- return "X86 Load Value Injection (LVI) Load Hardening (Unoptimized)";
- }
- bool runOnMachineFunction(MachineFunction &MF) override;
- static char ID;
-};
-
-} // end anonymous namespace
-
-char X86LoadValueInjectionLoadHardeningUnoptimizedPass::ID = 0;
-
-bool X86LoadValueInjectionLoadHardeningUnoptimizedPass::runOnMachineFunction(
- MachineFunction &MF) {
- LLVM_DEBUG(dbgs() << "***** " << getPassName() << " : " << MF.getName()
- << " *****\n");
- const X86Subtarget *STI = &MF.getSubtarget<X86Subtarget>();
- if (!STI->useLVILoadHardening())
- return false;
-
- // FIXME: support 32-bit
- if (!STI->is64Bit())
- report_fatal_error("LVI load hardening is only supported on 64-bit", false);
-
- // Don't skip functions with the "optnone" attr but participate in opt-bisect.
- const Function &F = MF.getFunction();
- if (!F.hasOptNone() && skipFunction(F))
- return false;
-
- bool Modified = false;
- ++NumFunctionsConsidered;
-
- const TargetInstrInfo *TII = STI->getInstrInfo();
- for (auto &MBB : MF) {
- for (auto &MI : MBB) {
- if (!MI.mayLoad() || MI.getOpcode() == X86::LFENCE ||
- MI.getOpcode() == X86::MFENCE)
- continue;
-
- MachineBasicBlock::iterator InsertionPt =
- MI.getNextNode() ? MI.getNextNode() : MBB.end();
- BuildMI(MBB, InsertionPt, DebugLoc(), TII->get(X86::LFENCE));
- ++NumFences;
- Modified = true;
- }
- }
-
- if (Modified)
- ++NumFunctionsMitigated;
-
- return Modified;
-}
-
-INITIALIZE_PASS(X86LoadValueInjectionLoadHardeningUnoptimizedPass, PASS_KEY,
- "X86 LVI load hardening", false, false)
-
-FunctionPass *llvm::createX86LoadValueInjectionLoadHardeningUnoptimizedPass() {
- return new X86LoadValueInjectionLoadHardeningUnoptimizedPass();
-}
diff --git a/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp b/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp
--- a/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp
+++ b/llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp
@@ -22,6 +22,7 @@
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/Pass.h"
+#include "llvm/Target/TargetMachine.h"
using namespace llvm;
#define DEBUG_TYPE "x86-seses"
@@ -86,13 +87,19 @@
bool X86SpeculativeExecutionSideEffectSuppression::runOnMachineFunction(
MachineFunction &MF) {
- if (!EnableSpeculativeExecutionSideEffectSuppression)
+
+ const auto &OptLevel = MF.getTarget().getOptLevel();
+ const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
+
+ // Check whether SESES needs to run as the fallback for LVI at O0 or if the
+ // user explicitly passed the SESES flag.
+ if (!EnableSpeculativeExecutionSideEffectSuppression &&
+ !(Subtarget.useLVILoadHardening() && OptLevel == CodeGenOpt::None))
return false;
LLVM_DEBUG(dbgs() << "********** " << getPassName() << " : " << MF.getName()
<< " **********\n");
bool Modified = false;
- const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
const X86InstrInfo *TII = Subtarget.getInstrInfo();
for (MachineBasicBlock &MBB : MF) {
MachineInstr *FirstTerminator = nullptr;
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -489,10 +489,12 @@
void X86PassConfig::addPostRegAlloc() {
addPass(createX86FloatingPointStackifierPass());
+ // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
+ // to using the Speculative Execution Side Effect Suppression pass for
+ // mitigation. This is to prevent slow downs due to
+ // analyses needed by the LVIHardening pass when compiling at -O0.
if (getOptLevel() != CodeGenOpt::None)
addPass(createX86LoadValueInjectionLoadHardeningPass());
- else
- addPass(createX86LoadValueInjectionLoadHardeningUnoptimizedPass());
}
void X86PassConfig::addPreSched2() { addPass(createX86ExpandPseudoPass()); }
diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll
--- a/llvm/test/CodeGen/X86/O0-pipeline.ll
+++ b/llvm/test/CodeGen/X86/O0-pipeline.ll
@@ -46,7 +46,6 @@
; CHECK-NEXT: Fast Register Allocator
; CHECK-NEXT: Bundle Machine CFG Edges
; CHECK-NEXT: X86 FP Stackifier
-; CHECK-NEXT: X86 Load Value Injection (LVI) Load Hardening (Unoptimized)
; CHECK-NEXT: Fixup Statepoint Caller Saved
; CHECK-NEXT: Lazy Machine Block Frequency Analysis
; CHECK-NEXT: Machine Optimization Remark Emitter
diff --git a/llvm/test/CodeGen/X86/lvi-hardening-loads.ll b/llvm/test/CodeGen/X86/lvi-hardening-loads.ll
--- a/llvm/test/CodeGen/X86/lvi-hardening-loads.ll
+++ b/llvm/test/CodeGen/X86/lvi-hardening-loads.ll
@@ -26,10 +26,15 @@
; X64-NEXT: jmp .LBB0_1
; X64-NOOPT: # %bb.0: # %entry
+; X64-NOOPT-NEXT: lfence
; X64-NOOPT-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NOOPT-NEXT: lfence
; X64-NOOPT-NEXT: movl %esi, -{{[0-9]+}}(%rsp)
+; X64-NOOPT-NEXT: lfence
; X64-NOOPT-NEXT: movl $0, -{{[0-9]+}}(%rsp)
; X64-NOOPT-NEXT: lfence
+; X64-NOOPT-NEXT: lfence
+; X64-NOOPT-NEXT: lfence
; X64-NOOPT-NEXT: movl $0, -{{[0-9]+}}(%rsp)
for.cond: ; preds = %for.inc, %entry
@@ -48,6 +53,7 @@
; X64-NOOPT: .LBB0_1: # %for.cond
; X64-NOOPT-NEXT: # =>This Inner Loop Header: Depth=1
+; X64-NOOPT-NEXT: lfence
; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax
; X64-NOOPT-NEXT: lfence
; X64-NOOPT-NEXT: cmpl -{{[0-9]+}}(%rsp), %eax
@@ -73,12 +79,13 @@
; X64-NOOPT: # %bb.2: # %for.body
; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1
-; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax
; X64-NOOPT-NEXT: lfence
+; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax
; X64-NOOPT-NEXT: cltd
; X64-NOOPT-NEXT: movl $2, %ecx
; X64-NOOPT-NEXT: idivl %ecx
; X64-NOOPT-NEXT: cmpl $0, %edx
+; X64-NOOPT-NEXT: lfence
; X64-NOOPT-NEXT: jne .LBB0_4
if.then: ; preds = %for.body
@@ -105,6 +112,7 @@
; X64-NOOPT: # %bb.3: # %if.then
; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1
+; X64-NOOPT-NEXT: lfence
; X64-NOOPT-NEXT: movq -{{[0-9]+}}(%rsp), %rax
; X64-NOOPT-NEXT: lfence
; X64-NOOPT-NEXT: movslq -{{[0-9]+}}(%rsp), %rcx
@@ -126,10 +134,12 @@
; X64-NOOPT: .LBB0_5: # %for.inc
; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1
-; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax
; X64-NOOPT-NEXT: lfence
+; X64-NOOPT-NEXT: movl -{{[0-9]+}}(%rsp), %eax
; X64-NOOPT-NEXT: addl $1, %eax
+; X64-NOOPT-NEXT: lfence
; X64-NOOPT-NEXT: movl %eax, -{{[0-9]+}}(%rsp)
+; X64-NOOPT-NEXT: lfence
; X64-NOOPT-NEXT: jmp .LBB0_1
for.end: ; preds = %for.cond