Enable lowering of llvm.speculation_safe_value to DSB/ISB pair.

[kristof.beyls]

Summary

This is part of implementing a technique to mitigate against Spectre v1,
similar in spirit to what has been proposed by Chandler for X86_64 at
http://lists.llvm.org/pipermail/llvm-dev/2018-March/122085.html.

This patch enables not tracking miss-speculation status, but instead
using a more heavy weight DSB + ISB barrier to implement the semantics
of the llvm.speculation_safe_value intrinsic. This may be beneficial
e.g. when only an infrequently executed part of the code needs to be
protected using manually inserted intrinsics.

lib/Target/AArch64/AArch64InstrInfo.cpp

if (AArch64GenInstrInfo::isSchedulingBoundary(MI, MBB, MF))
     return true;
 
   switch (MI.getOpcode()) {
   case AArch64::HINT:
     // CSDB hints are scheduling barriers.
     if (MI.getOperand(0).getImm() == 0x14)
       return true;
     break;
+  case AArch64::DSB:
+  case AArch64::ISB:
+    // DSB and ISB also are scheduling barriers.
+    return true;
   default:
     ;
   }
 
   return false;
 }

lib/Target/AArch64/AArch64SpeculationHardening.cpp

case AArch64::SpeculationSafeValueX:
     for (MachineOperand Op : MI.defs()) {
       assert(Op.isReg());
       // Mark this register and all its super-registers as needing to be
       // value speculation protected before its next use, by using a CSDB
       // barrier instruction.
       TRI->markSuperRegs(RegsNeedingBarrierBeforeUse, Op.getReg());
     }
 
-    assert(EnableSpeculationTracking);
+    if (EnableSpeculationTracking) {
       // mask off with CFST state
       BuildMI(MBB, MBBI, MI.getDebugLoc(),
               Is64Bit ? TII->get(AArch64::ANDXrs) : TII->get(AArch64::ANDWrs))
           .addDef(DstReg)
           .addUse(SrcReg, RegState::Kill)
           .addUse(Is64Bit ? MisspeculatingTaintVR : MisspeculatingTaintVR32Bit)
           .addImm(0);
+    } else {
+      // Nothing to do: the destination register has been marked as needing to
+      // be speculation protected, and since we don't use control flow
+      // speculation tracking a full control flow speculation barrier will be
+      // emitted.
+      if (DstReg != SrcReg) {
+        BuildMI(MBB, MBBI, MI.getDebugLoc(),
+                Is64Bit ? TII->get(AArch64::ORRXrs) : TII->get(AArch64::ORRWrs))
+            .addDef(DstReg)
+            .addUse(Is64Bit ? AArch64::XZR : AArch64::WZR)
+            .addUse(SrcReg, RegState::Kill)
+            .addImm(0);
+      }
+    }
     MI.eraseFromParent();
     return true;
   }
   return false;
 }
 
 bool AArch64SpeculationHardening::insertSpeculationBarrier(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL) {
-  assert(EnableSpeculationTracking && "Speculation tracking must be enabled "
-                                      "when inserting speculation barriers");
+  if (EnableSpeculationTracking) {
     // insert data value speculation barrier (CSDB)
     BuildMI(MBB, MBBI, DL, TII->get(AArch64::HINT)).addImm(0x14);
+  } else {
+    // insert full control flow speculation barrier (DSB SYS + ISB)
+    BuildMI(MBB, MBBI, DL, TII->get(AArch64::DSB)).addImm(0xf);
+    BuildMI(MBB, MBBI, DL, TII->get(AArch64::ISB)).addImm(0xf);
+  }
   RegsNeedingBarrierBeforeUse.reset();
   return true;
 }
 
 bool AArch64SpeculationHardening::lowerSpeculationSafeValuePseudo(
     MachineBasicBlock &MBB) {
   bool Modified = false;
 
   RegsNeedingBarrierBeforeUse.reset();
 
   // The following loop iterates over all instructions in the basic block, and
   // performs 2 operations:
   // 1. Insert a CSDB at this location if needed.
   // 2. Insert a masking instruction (AND) with the CFST to lower the
   //    ValueSpeculationSafe pseudo instruction.
   //
   // The insertion of the CSDB is done as late as possible (i.e. just before
   // the use of a masked register), in the hope that that will reduce the total
   // number of CSDBs in a block when there are multiple masked registers in the
   // block.
+  //
+  // In case control flow speculation tracking is not enabled, obviously, no
+  // masking of registers can happen. Instead, a full control flow speculation
+  // barrier will be generated in the locations where otherwise the limited
+  // data value speculation barrier (CSDB) would be inserted.
   MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
   DebugLoc DL;
   while (MBBI != E) {
     MachineInstr &MI = *MBBI;
     DL = MI.getDebugLoc();
     MachineBasicBlock::iterator NMBBI = std::next(MBBI);
 
     // First check if a barrier needs to be inserted due to earlier registers

test/CodeGen/AArch64/speculation-hardening.ll

 ; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -aarch64-track-speculation | FileCheck %s --check-prefix=CFST
+; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu | FileCheck %s --check-prefix=DSBISB
 
 declare i8 @llvm.speculationsafevalue.i8(i8)
 declare i96 @llvm.speculationsafevalue.i96(i96)
 declare i8* @llvm.speculationsafevalue.p0(i8*)
 
 declare void @function_without_arguments()
 
 define i32 @f(i8* nocapture readonly %p, i32 %i, i32 %N) local_unnamed_addr {

 ; CFST: csel x16, x16, xzr, lt
   %idxprom = sext i32 %i to i64
   %arrayidx = getelementptr inbounds i8, i8* %p, i64 %idxprom
   %0 = load i8, i8* %arrayidx, align 1
   %1 = tail call i8 @llvm.speculationsafevalue.i8(i8 %0)
 ; CFST:      ldrb [[LOADED:w[0-9]+]],
 ; CFST:      and  [[DST:w[0-9]+]], [[LOADED]], w16
 ; CFST-NEXT: csdb
+; DSBISB:      ldrb [[LOADED:w[0-9]+]],
+; DSBISB-NEXT: dsb sy
+; DSBISB-NEXT: isb
   %conv = zext i8 %1 to i32
   br label %return
 
 ; CFST: csel x16, x16, xzr, ge
 return:                                           ; preds = %entry, %if.then
   %retval.0 = phi i32 [ %conv, %if.then ], [ 0, %entry ]
 ; CFST:  mov x17, sp
 ; CFST:  and x17, x17, x16
 ; CFST:  mov sp, x17
   ret i32 %retval.0
 }
 
 define i64 @nonpow2_narrowing(i128* %p) {
 ; CFST-LABEL:  nonpow2_narrowing:
   %a = load i128, i128* %p
 ; CFST: ldp [[SRC1:x[0-9]+]], [[SRC2:x[0-9]+]], [x0]
   %b = trunc i128 %a to i96
   %c = tail call i96 @llvm.speculationsafevalue.i96(i96 %b)
 ; CFST: and [[SRC1]], [[SRC1]], x16
 ; CFST-NOT: csdb
 ; CFST: and [[SRC2]], [[SRC2]], x16
 ; CFST-NEXT: csdb
 ; CFST: add x0, 
+; DSBISB:      dsb sy
+; DSBISB-NEXT: isb
+; DSBISB-NOT:  dsb sy
+; DSBISB-NOT:  isb
+; DSBISB-NEXT: add x0, 
   %lo = trunc i96 %c to i64
   %hi96 = lshr i96 %c, 64
   %hi = trunc i96 %hi96 to i64
   %d = add i64 %lo, %hi
   ret i64 %d
 }
 
 define i8* @ptr(i8* %p) {
 ; CFST-LABEL:  ptr:
   %a = tail call i8* @llvm.speculationsafevalue.p0(i8* %p)
 ; CFST: and x0, x0, x16
 ; CFST: csdb
 ; CFST: ret
+; DSBISB:      dsb sy
+; DSBISB-NEXT: isb
+; DSBISB-NEXT: ret
   ret i8* %a
 }
 
 define i8* @csdb_before_call(i8* %p) {
 ; CFST-LABEL:  csdb_before_call:
   %a = tail call i8* @llvm.speculationsafevalue.p0(i8* %p)
   call void @function_without_arguments()
   ret i8* %a
 ; CFST: and [[DST:x[0-9]+]], x0, x16
 ; CFST: csdb
 ; CFST: bl function_without_arguments
+; DSBISB:      dsb sy
+; DSBISB-NEXT: isb
+; DSBISB-NEXT: bl function_without_arguments
 ; CFST: ret
 }
 
 define i8* @csdb_on_unused_in_bb_with_terminator(i8* %p, i32 %v1, i32 %v2) {
 ; CFST-LABEL:  csdb_on_unused_in_bb_with_terminator:
 entry:
   %a = tail call i8* @llvm.speculationsafevalue.p0(i8* %p)
   %cmp = icmp slt i32 %v1, %v2
   br i1 %cmp, label %if.then, label %return
 ; CFST: and [[DST:x[0-9]+]], x0, x16
 ; CFST: csdb
 ; CFST: b.ge
 ; CFST-NOT: csdb
+; DSBISB:      dsb sy
+; DSBISB-NEXT: isb
+; DSBISB-NEXT: b.ge
 if.then:
   %tmp1 = sdiv i32 %v1, %v2
   %b = getelementptr i8, i8* %a, i32 %tmp1
   br label %return
 
 return:
   %c = phi i8* [ %a, %entry ] , [ %b, %if.then ]
   ret i8* %c
if.then:
   %tmp1 = sdiv i32 %v1, %v2
   %b1 = getelementptr i8, i8* %p, i32 %tmp1
   %b = tail call i8* @llvm.speculationsafevalue.p0(i8* %b1)
   br label %return
 ; CFST: and [[DST:x[0-9]+]], [[SRC:x[0-9]+]], x16
 ; CFST: csdb
 ; CFST-NOT: csdb
 ; CFST: ret
+; DSBISB:      dsb sy
+; DSBISB-NEXT: isb
+; DSBISB-NEXT: .LBB{{[0-9_]+}}: // %return
+; DSBISB-NEXT: ret
  
 return:
   %c = phi i8* [ %p, %entry ] , [ %b, %if.then ]
   ret i8* %c
 }
 
 ; Make sure that for a tail call, taint doesn't get put into SP twice.
 define i32 @tail_caller(i32 %a) local_unnamed_addr {