[x86][lvi][seses] Use SESES at O0 for LVI mitigation

[zbrid]

Summary

Use SESES as the fallback at O0 where the optimized LVI pass isn't desired due
to its effect on build times at O0.

I updated the LVI tests since this changes the code gen for the tests touched in the parent revision.

This is a follow up to the comments I made here: https://reviews.llvm.org/D80964

Hopefully we can continue the discussion here.

[zbrid]

llvm/test/CodeGen/X86/lvi-hardening-loads.ll

37

Sorry, this is totally weird and I'll look into why this happens.

[zbrid]

Add comment to clarify

[zbrid]

@sconstab @craig.topper - Ping for review

[sconstab]

Other than the inline comments, LGTM.

llvm/lib/Target/X86/X86TargetMachine.cpp

497–498

Please add a comment here to clarify that if getOptLevel() == CodeGenOpt::None then SESES will provide mitigation when LVILoadHardening is enabled.

llvm/test/CodeGen/X86/lvi-hardening-loads.ll

37

Has this been fixed?

[zbrid]

Add inline comment about lvi hardening falling back to seses

[zbrid]

Fix redundant LFENCE problem with SESES

[zbrid]

Going to merge this today. I updated based on the comments.

[zbrid]

Update commit message to add info about SESES change too

llvm/lib/Target/X86/X86.h

 /// ways.
 FunctionPass *createX86PartialReductionPass();
 
 InstructionSelector *createX86InstructionSelector(const X86TargetMachine &TM,
                                                   X86Subtarget &,
                                                   X86RegisterBankInfo &);
 
 FunctionPass *createX86LoadValueInjectionLoadHardeningPass();
-FunctionPass *createX86LoadValueInjectionLoadHardeningUnoptimizedPass();
 FunctionPass *createX86LoadValueInjectionRetHardeningPass();
 FunctionPass *createX86SpeculativeLoadHardeningPass();
 FunctionPass *createX86SpeculativeExecutionSideEffectSuppression();
 
 void initializeEvexToVexInstPassPass(PassRegistry &);
 void initializeFixupBWInstPassPass(PassRegistry &);
 void initializeFixupLEAPassPass(PassRegistry &);
 void initializeFPSPass(PassRegistry &);
 void initializeWinEHStatePassPass(PassRegistry &);
 void initializeX86AvoidSFBPassPass(PassRegistry &);
 void initializeX86AvoidTrailingCallPassPass(PassRegistry &);
 void initializeX86CallFrameOptimizationPass(PassRegistry &);
 void initializeX86CmovConverterPassPass(PassRegistry &);
 void initializeX86CondBrFoldingPassPass(PassRegistry &);
 void initializeX86DomainReassignmentPass(PassRegistry &);
 void initializeX86ExecutionDomainFixPass(PassRegistry &);
 void initializeX86ExpandPseudoPass(PassRegistry &);
 void initializeX86FixupSetCCPassPass(PassRegistry &);
 void initializeX86FlagsCopyLoweringPassPass(PassRegistry &);
-void initializeX86LoadValueInjectionLoadHardeningUnoptimizedPassPass(PassRegistry &);
 void initializeX86LoadValueInjectionLoadHardeningPassPass(PassRegistry &);
 void initializeX86LoadValueInjectionRetHardeningPassPass(PassRegistry &);
 void initializeX86OptimizeLEAPassPass(PassRegistry &);
 void initializeX86PartialReductionPass(PassRegistry &);
 void initializeX86SpeculativeLoadHardeningPassPass(PassRegistry &);
 void initializeX86SpeculativeExecutionSideEffectSuppressionPass(PassRegistry &);
 
 namespace X86AS {

llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp

 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
 INITIALIZE_PASS_DEPENDENCY(MachineDominanceFrontier)
 INITIALIZE_PASS_END(X86LoadValueInjectionLoadHardeningPass, PASS_KEY,
                     "X86 LVI load hardening", false, false)
 
 FunctionPass *llvm::createX86LoadValueInjectionLoadHardeningPass() {
   return new X86LoadValueInjectionLoadHardeningPass();
 }
-
-namespace {
-
-/// The `X86LoadValueInjectionLoadHardeningPass` above depends on expensive
-/// analysis passes that add complexity to the pipeline. This complexity
-/// can cause noticable overhead when no optimizations are enabled, i.e., -O0.
-/// The purpose of `X86LoadValueInjectionLoadHardeningUnoptimizedPass` is to
-/// provide the same security as the optimized pass, but without adding
-/// unnecessary complexity to the LLVM pipeline.
-///
-/// The behavior of this pass is simply to insert an LFENCE after every load
-/// instruction.
-class X86LoadValueInjectionLoadHardeningUnoptimizedPass
-    : public MachineFunctionPass {
-public:
-  X86LoadValueInjectionLoadHardeningUnoptimizedPass()
-      : MachineFunctionPass(ID) {}
-
-  StringRef getPassName() const override {
-    return "X86 Load Value Injection (LVI) Load Hardening (Unoptimized)";
-  }
-  bool runOnMachineFunction(MachineFunction &MF) override;
-  static char ID;
-};
-
-} // end anonymous namespace
-
-char X86LoadValueInjectionLoadHardeningUnoptimizedPass::ID = 0;
-
-bool X86LoadValueInjectionLoadHardeningUnoptimizedPass::runOnMachineFunction(
-    MachineFunction &MF) {
-  LLVM_DEBUG(dbgs() << "***** " << getPassName() << " : " << MF.getName()
-                    << " *****\n");
-  const X86Subtarget *STI = &MF.getSubtarget<X86Subtarget>();
-  if (!STI->useLVILoadHardening())
-    return false;
-
-  // FIXME: support 32-bit
-  if (!STI->is64Bit())
-    report_fatal_error("LVI load hardening is only supported on 64-bit", false);
-
-  // Don't skip functions with the "optnone" attr but participate in opt-bisect.
-  const Function &F = MF.getFunction();
-  if (!F.hasOptNone() && skipFunction(F))
-    return false;
-
-  bool Modified = false;
-  ++NumFunctionsConsidered;
-
-  const TargetInstrInfo *TII = STI->getInstrInfo();
-  for (auto &MBB : MF) {
-    for (auto &MI : MBB) {
-      if (!MI.mayLoad() || MI.getOpcode() == X86::LFENCE ||
-          MI.getOpcode() == X86::MFENCE)
-        continue;
-
-      MachineBasicBlock::iterator InsertionPt =
-          MI.getNextNode() ? MI.getNextNode() : MBB.end();
-      BuildMI(MBB, InsertionPt, DebugLoc(), TII->get(X86::LFENCE));
-      ++NumFences;
-      Modified = true;
-    }
-  }
-
-  if (Modified)
-    ++NumFunctionsMitigated;
-
-  return Modified;
-}
-
-INITIALIZE_PASS(X86LoadValueInjectionLoadHardeningUnoptimizedPass, PASS_KEY,
-                "X86 LVI load hardening", false, false)
-
-FunctionPass *llvm::createX86LoadValueInjectionLoadHardeningUnoptimizedPass() {
-  return new X86LoadValueInjectionLoadHardeningUnoptimizedPass();
-}

llvm/lib/Target/X86/X86SpeculativeExecutionSideEffectSuppression.cpp

 #include "X86.h"
 #include "X86InstrInfo.h"
 #include "X86Subtarget.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/Pass.h"
+#include "llvm/Target/TargetMachine.h"
 using namespace llvm;
 
 #define DEBUG_TYPE "x86-seses"
 
 STATISTIC(NumLFENCEsInserted, "Number of lfence instructions inserted");
 
 static cl::opt<bool> EnableSpeculativeExecutionSideEffectSuppression(
     "x86-seses-enable",
static bool hasConstantAddressingMode(const MachineInstr &MI) {
   for (const MachineOperand &MO : MI.uses())
     if (MO.isReg() && X86::RIP != MO.getReg())
       return false;
   return true;
 }
 
 bool X86SpeculativeExecutionSideEffectSuppression::runOnMachineFunction(
     MachineFunction &MF) {
-  if (!EnableSpeculativeExecutionSideEffectSuppression)
+
+  const auto &OptLevel = MF.getTarget().getOptLevel();
+  const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
+
+  // Check whether SESES needs to run as the fallback for LVI at O0 or if the
+  // user explicitly passed the SESES flag.
+  if (!EnableSpeculativeExecutionSideEffectSuppression &&
+      !(Subtarget.useLVILoadHardening() && OptLevel == CodeGenOpt::None))
     return false;
 
   LLVM_DEBUG(dbgs() << "********** " << getPassName() << " : " << MF.getName()
                     << " **********\n");
   bool Modified = false;
-  const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
   const X86InstrInfo *TII = Subtarget.getInstrInfo();
   for (MachineBasicBlock &MBB : MF) {
     MachineInstr *FirstTerminator = nullptr;
-
+    // Keep track of whether the previous instruction was an LFENCE to avoid
+    // adding redundant LFENCEs.
+    bool PrevInstIsLFENCE = false;
     for (auto &MI : MBB) {
+
+      if (MI.getOpcode() == X86::LFENCE) {
+        PrevInstIsLFENCE = true;
+        continue;
+      }
       // We want to put an LFENCE before any instruction that
       // may load or store. This LFENCE is intended to avoid leaking any secret
       // data due to a given load or store. This results in closing the cache
       // and memory timing side channels. We will treat terminators that load
       // or store separately.
       if (MI.mayLoadOrStore() && !MI.isTerminator()) {
+        if (!PrevInstIsLFENCE) {
           BuildMI(MBB, MI, DebugLoc(), TII->get(X86::LFENCE));
           NumLFENCEsInserted++;
           Modified = true;
+        }
         if (OneLFENCEPerBasicBlock)
           break;
       }
       // The following section will be LFENCEing before groups of terminators
       // that include branches. This will close the branch prediction side
       // channels since we will prevent code executing after misspeculation as
       // a result of the LFENCEs placed with this logic.
 
       // Keep track of the first terminator in a basic block since if we need
       // to LFENCE the terminators in this basic block we must add the
       // instruction before the first terminator in the basic block (as
       // opposed to before the terminator that indicates an LFENCE is
       // required). An example of why this is necessary is that the
       // X86InstrInfo::analyzeBranch method assumes all terminators are grouped
       // together and terminates it's analysis once the first non-termintor
       // instruction is found.
       if (MI.isTerminator() && FirstTerminator == nullptr)
         FirstTerminator = &MI;
 
       // Look for branch instructions that will require an LFENCE to be put
       // before this basic block's terminators.
-      if (!MI.isBranch() || OmitBranchLFENCEs)
+      if (!MI.isBranch() || OmitBranchLFENCEs) {
         // This isn't a branch or we're not putting LFENCEs before branches.
+        PrevInstIsLFENCE = false;
         continue;
+      }
 
-      if (OnlyLFENCENonConst && hasConstantAddressingMode(MI))
+      if (OnlyLFENCENonConst && hasConstantAddressingMode(MI)) {
         // This is a branch, but it only has constant addressing mode and we're
         // not adding LFENCEs before such branches.
+        PrevInstIsLFENCE = false;
         continue;
+      }
 
       // This branch requires adding an LFENCE.
+      if (!PrevInstIsLFENCE) {
         BuildMI(MBB, FirstTerminator, DebugLoc(), TII->get(X86::LFENCE));
         NumLFENCEsInserted++;
         Modified = true;
+      }
       break;
     }
   }
 
   return Modified;
 }
 
 FunctionPass *llvm::createX86SpeculativeExecutionSideEffectSuppression() {
   return new X86SpeculativeExecutionSideEffectSuppression();
 }
 
 INITIALIZE_PASS(X86SpeculativeExecutionSideEffectSuppression, "x86-seses",
                 "X86 Speculative Execution Side Effect Suppression", false,
                 false)

llvm/lib/Target/X86/X86TargetMachine.cpp

 }
 void X86PassConfig::addMachineSSAOptimization() {
   addPass(createX86DomainReassignmentPass());
   TargetPassConfig::addMachineSSAOptimization();
 }
 
 void X86PassConfig::addPostRegAlloc() {
   addPass(createX86FloatingPointStackifierPass());
+  // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
+  // to using the Speculative Execution Side Effect Suppression pass for
+  // mitigation. This is to prevent slow downs due to
+  // analyses needed by the LVIHardening pass when compiling at -O0.
   if (getOptLevel() != CodeGenOpt::None)
     addPass(createX86LoadValueInjectionLoadHardeningPass());
-  else
-    addPass(createX86LoadValueInjectionLoadHardeningUnoptimizedPass());
 }
 
 void X86PassConfig::addPreSched2() { addPass(createX86ExpandPseudoPass()); }
 
 void X86PassConfig::addPreEmitPass() {
   if (getOptLevel() != CodeGenOpt::None) {
     addPass(new X86ExecutionDomainFix());
     addPass(createBreakFalseDeps());
   }

llvm/test/CodeGen/X86/O0-pipeline.ll

 ; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       X86 EFLAGS copy lowering
 ; CHECK-NEXT:       X86 WinAlloca Expander
 ; CHECK-NEXT:       Eliminate PHI nodes for register allocation
 ; CHECK-NEXT:       Two-Address instruction pass
 ; CHECK-NEXT:       Fast Register Allocator
 ; CHECK-NEXT:       Bundle Machine CFG Edges
 ; CHECK-NEXT:       X86 FP Stackifier
-; CHECK-NEXT:       X86 Load Value Injection (LVI) Load Hardening (Unoptimized)
 ; CHECK-NEXT:       Fixup Statepoint Caller Saved
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       Prologue/Epilogue Insertion & Frame Finalization
 ; CHECK-NEXT:       Post-RA pseudo instruction expansion pass
 ; CHECK-NEXT:       X86 pseudo instruction expansion pass
 ; CHECK-NEXT:       Analyze Machine Code For Garbage Collection
 ; CHECK-NEXT:       Insert fentry calls

llvm/test/CodeGen/X86/lvi-hardening-loads.ll

 ; X64-NEXT:      movq %rdi, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:      movl %esi, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:      movl $0, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:      lfence
 ; X64-NEXT:      movl $0, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:      jmp .LBB0_1
 
 ; X64-NOOPT: # %bb.0: # %entry
+; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movl %esi, -{{[0-9]+}}(%rsp)
+; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movl $0, -{{[0-9]+}}(%rsp)
 ; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movl $0, -{{[0-9]+}}(%rsp)
 
 for.cond:                                         ; preds = %for.inc, %entry
   %0 = load i32, i32* %i, align 4
   %1 = load i32, i32* %secret_size.addr, align 4
   %cmp = icmp slt i32 %0, %1
   br i1 %cmp, label %for.body, label %for.end
 
 ; X64: .LBB0_1: # %for.cond
 ; X64-NEXT:      # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:      movl -{{[0-9]+}}(%rsp), %eax
 ; X64-ALL-NEXT:  lfence
 ; X64-NEXT:      cmpl -{{[0-9]+}}(%rsp), %eax
 ; X64-ALL-NEXT:  lfence
 ; X64-NEXT:      jge .LBB0_5
 
 ; X64-NOOPT: .LBB0_1: # %for.cond
 ; X64-NOOPT-NEXT:      # =>This Inner Loop Header: Depth=1
+; X64-NOOPT-NEXT:  lfence
 ; X64-NOOPT-NEXT:      movl -{{[0-9]+}}(%rsp), %eax
 ; X64-NOOPT-NEXT:  lfence
 ; X64-NOOPT-NEXT:      cmpl -{{[0-9]+}}(%rsp), %eax
 ; X64-NOOPT-NEXT:  lfence
 ; X64-NOOPT-NEXT:      jge .LBB0_6
 
 for.body:                                         ; preds = %for.cond
   %2 = load i32, i32* %i, align 4

 ; X64-NEXT:      shrl $31, %ecx
 ; X64-NEXT:      addl %eax, %ecx
 ; X64-NEXT:      andl $-2, %ecx
 ; X64-NEXT:      cmpl %ecx, %eax
 ; X64-NEXT:      jne .LBB0_4
 
 ; X64-NOOPT: # %bb.2: # %for.body
 ; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1
-; X64-NOOPT-NEXT:      movl -{{[0-9]+}}(%rsp), %eax
 ; X64-NOOPT-NEXT:  lfence
+; X64-NOOPT-NEXT:      movl -{{[0-9]+}}(%rsp), %eax
 ; X64-NOOPT-NEXT:      cltd
 ; X64-NOOPT-NEXT:      movl $2, %ecx
 ; X64-NOOPT-NEXT:      idivl %ecx
 ; X64-NOOPT-NEXT:      cmpl $0, %edx
+; X64-NOOPT-NEXT:  lfence
 ; X64-NOOPT-NEXT:      jne .LBB0_4
 
 if.then:                                          ; preds = %for.body
   %3 = load i32**, i32*** %secret.addr, align 8
   %4 = load i32, i32* %ret_val, align 4
   %idxprom = sext i32 %4 to i64
   %arrayidx = getelementptr inbounds i32*, i32** %3, i64 %idxprom
   %5 = load i32*, i32** %arrayidx, align 8

 ; X64-NEXT:      movq (%rax,%rcx,8), %rax
 ; X64-NEXT:      lfence
 ; X64-NEXT:      movl (%rax), %eax
 ; X64-NEXT:      movl %eax, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:      jmp .LBB0_4
 
 ; X64-NOOPT: # %bb.3: # %if.then
 ; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1
+; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movq -{{[0-9]+}}(%rsp), %rax
 ; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movslq -{{[0-9]+}}(%rsp), %rcx
 ; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movq (%rax,%rcx,8), %rax
 ; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movl (%rax), %eax
 ; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movl %eax, -{{[0-9]+}}(%rsp)
 
 if.end:                                           ; preds = %if.then, %for.body
   br label %for.inc
 
 for.inc:                                          ; preds = %if.end
   %7 = load i32, i32* %i, align 4
   %inc = add nsw i32 %7, 1
   store i32 %inc, i32* %i, align 4
   br label %for.cond
 
 ; X64-NOOPT: .LBB0_5: # %for.inc
 ; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1
-; X64-NOOPT-NEXT:      movl -{{[0-9]+}}(%rsp), %eax
 ; X64-NOOPT-NEXT:      lfence
+; X64-NOOPT-NEXT:      movl -{{[0-9]+}}(%rsp), %eax
 ; X64-NOOPT-NEXT:      addl $1, %eax
+; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      movl %eax, -{{[0-9]+}}(%rsp)
+; X64-NOOPT-NEXT:      lfence
 ; X64-NOOPT-NEXT:      jmp .LBB0_1
 
 for.end:                                          ; preds = %for.cond
   %8 = load i32, i32* %ret_val, align 4
   ret i32 %8
 }
 
 ; Function Attrs: nounwind
 declare void @llvm.x86.sse2.lfence() #1
 
 attributes #0 = { "target-features"="+lvi-load-hardening" }
 attributes #1 = { nounwind }