Index: lib/Target/AArch64/AArch64InstrInfo.cpp =================================================================== --- lib/Target/AArch64/AArch64InstrInfo.cpp +++ lib/Target/AArch64/AArch64InstrInfo.cpp @@ -5462,6 +5462,10 @@ if (MI.getOperand(0).getImm() == 0x14) return true; break; + case AArch64::DSB: + case AArch64::ISB: + // DSB and ISB also are scheduling barriers. + return true; default: ; } Index: lib/Target/AArch64/AArch64SpeculationHardening.cpp =================================================================== --- lib/Target/AArch64/AArch64SpeculationHardening.cpp +++ lib/Target/AArch64/AArch64SpeculationHardening.cpp @@ -335,14 +335,28 @@ TRI->markSuperRegs(RegsNeedingBarrierBeforeUse, Op.getReg()); } - assert(EnableSpeculationTracking); - // mask off with CFST state - BuildMI(MBB, MBBI, MI.getDebugLoc(), - Is64Bit ? TII->get(AArch64::ANDXrs) : TII->get(AArch64::ANDWrs)) - .addDef(DstReg) - .addUse(SrcReg, RegState::Kill) - .addUse(Is64Bit ? MisspeculatingTaintVR : MisspeculatingTaintVR32Bit) - .addImm(0); + if (EnableSpeculationTracking) { + // mask off with CFST state + BuildMI(MBB, MBBI, MI.getDebugLoc(), + Is64Bit ? TII->get(AArch64::ANDXrs) : TII->get(AArch64::ANDWrs)) + .addDef(DstReg) + .addUse(SrcReg, RegState::Kill) + .addUse(Is64Bit ? MisspeculatingTaintVR : MisspeculatingTaintVR32Bit) + .addImm(0); + } else { + // Nothing to do: the destination register has been marked as needing to + // be speculation protected, and since we don't use control flow + // speculation tracking a full control flow speculation barrier will be + // emitted. + if (DstReg != SrcReg) { + BuildMI(MBB, MBBI, MI.getDebugLoc(), + Is64Bit ? TII->get(AArch64::ORRXrs) : TII->get(AArch64::ORRWrs)) + .addDef(DstReg) + .addUse(Is64Bit ? AArch64::XZR : AArch64::WZR) + .addUse(SrcReg, RegState::Kill) + .addImm(0); + } + } MI.eraseFromParent(); return true; } @@ -351,10 +365,14 @@ bool AArch64SpeculationHardening::insertSpeculationBarrier( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL) { - assert(EnableSpeculationTracking && "Speculation tracking must be enabled " - "when inserting speculation barriers"); - // insert data value speculation barrier (CSDB) - BuildMI(MBB, MBBI, DL, TII->get(AArch64::HINT)).addImm(0x14); + if (EnableSpeculationTracking) { + // insert data value speculation barrier (CSDB) + BuildMI(MBB, MBBI, DL, TII->get(AArch64::HINT)).addImm(0x14); + } else { + // insert full control flow speculation barrier (DSB SYS + ISB) + BuildMI(MBB, MBBI, DL, TII->get(AArch64::DSB)).addImm(0xf); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ISB)).addImm(0xf); + } RegsNeedingBarrierBeforeUse.reset(); return true; } @@ -375,6 +393,11 @@ // the use of a masked register), in the hope that that will reduce the total // number of CSDBs in a block when there are multiple masked registers in the // block. + // + // In case control flow speculation tracking is not enabled, obviously, no + // masking of registers can happen. Instead, a full control flow speculation + // barrier will be generated in the locations where otherwise the limited + // data value speculation barrier (CSDB) would be inserted. MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); DebugLoc DL; while (MBBI != E) { Index: test/CodeGen/AArch64/speculation-hardening.ll =================================================================== --- test/CodeGen/AArch64/speculation-hardening.ll +++ test/CodeGen/AArch64/speculation-hardening.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -aarch64-track-speculation | FileCheck %s --check-prefix=CFST +; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu | FileCheck %s --check-prefix=DSBISB declare i8 @llvm.speculationsafevalue.i8(i8) declare i96 @llvm.speculationsafevalue.i96(i96) @@ -31,6 +32,9 @@ ; CFST: ldrb [[LOADED:w[0-9]+]], ; CFST: and [[DST:w[0-9]+]], [[LOADED]], w16 ; CFST-NEXT: csdb +; DSBISB: ldrb [[LOADED:w[0-9]+]], +; DSBISB-NEXT: dsb sy +; DSBISB-NEXT: isb %conv = zext i8 %1 to i32 br label %return @@ -54,6 +58,11 @@ ; CFST: and [[SRC2]], [[SRC2]], x16 ; CFST-NEXT: csdb ; CFST: add x0, +; DSBISB: dsb sy +; DSBISB-NEXT: isb +; DSBISB-NOT: dsb sy +; DSBISB-NOT: isb +; DSBISB-NEXT: add x0, %lo = trunc i96 %c to i64 %hi96 = lshr i96 %c, 64 %hi = trunc i96 %hi96 to i64 @@ -67,6 +76,9 @@ ; CFST: and x0, x0, x16 ; CFST: csdb ; CFST: ret +; DSBISB: dsb sy +; DSBISB-NEXT: isb +; DSBISB-NEXT: ret ret i8* %a } @@ -78,6 +90,9 @@ ; CFST: and [[DST:x[0-9]+]], x0, x16 ; CFST: csdb ; CFST: bl function_without_arguments +; DSBISB: dsb sy +; DSBISB-NEXT: isb +; DSBISB-NEXT: bl function_without_arguments ; CFST: ret } @@ -91,6 +106,9 @@ ; CFST: csdb ; CFST: b.ge ; CFST-NOT: csdb +; DSBISB: dsb sy +; DSBISB-NEXT: isb +; DSBISB-NEXT: b.ge if.then: %tmp1 = sdiv i32 %v1, %v2 %b = getelementptr i8, i8* %a, i32 %tmp1 @@ -117,6 +135,10 @@ ; CFST: csdb ; CFST-NOT: csdb ; CFST: ret +; DSBISB: dsb sy +; DSBISB-NEXT: isb +; DSBISB-NEXT: .LBB{{[0-9_]+}}: // %return +; DSBISB-NEXT: ret return: %c = phi i8* [ %p, %entry ] , [ %b, %if.then ]