diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def --- a/clang/include/clang/Basic/Builtins.def +++ b/clang/include/clang/Basic/Builtins.def @@ -1533,6 +1533,9 @@ BUILTIN(__builtin_ms_va_end, "vc*&", "n") BUILTIN(__builtin_ms_va_copy, "vc*&c*&", "n") +// T __builtin_speculation_safe_value (T val) +BUILTIN(__builtin_speculation_safe_value, "vv", "t") + #undef BUILTIN #undef LIBBUILTIN #undef LANGBUILTIN diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -9556,6 +9556,10 @@ "the type is not trivially copyable|" "the type does not have the expected form}1">; +def err_specsafevalue_builtin_must_be_pointer_or_integral : Error< + "argument to speculation_safe_value builtin must be a pointer or integer " + "(%0 invalid)">; + def warn_dereference_of_noderef_type : Warning< "dereferencing %0; was declared with a 'noderef' type">, InGroup; def warn_dereference_of_noderef_type_no_decl : Warning< diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -10717,6 +10717,8 @@ AtomicExpr::AtomicOp Op); ExprResult SemaBuiltinOperatorNewDeleteOverloaded(ExprResult TheCallResult, bool IsDelete); + ExprResult + SemaBuiltinSpeculationSafeValueOverloaded(ExprResult TheCallResult); bool SemaBuiltinConstantArg(CallExpr *TheCall, int ArgNum, llvm::APSInt &Result); bool SemaBuiltinConstantArgRange(CallExpr *TheCall, int ArgNum, int Low, diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -3980,6 +3980,16 @@ Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val"); return RValue::get(Builder.CreateStore(ArgPtr, DestAddr)); } + case Builtin::BI__builtin_speculation_safe_value: { + Value *Val = EmitScalarExpr(E->getArg(0)); + + llvm::Type *T = ConvertType(E->getType()); + assert((isa(T) || isa(T)) && + "unsupported type"); + + return RValue::get(Builder.CreateCall( + CGM.getIntrinsic(Intrinsic::speculationsafevalue, T), {Val})); + } } // If this is an alias for a lib function (e.g. __builtin_sin), emit diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp --- a/clang/lib/Frontend/InitPreprocessor.cpp +++ b/clang/lib/Frontend/InitPreprocessor.cpp @@ -1083,6 +1083,8 @@ Builder.defineMacro("__GLIBCXX_BITSIZE_INT_N_0", "128"); } + Builder.defineMacro("__HAVE_SPECULATION_SAFE_VALUE"); + // Get other target #defines. TI.getTargetDefines(LangOpts, Builder); } diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -1493,6 +1493,8 @@ if (SemaBuiltinOSLogFormat(TheCall)) return ExprError(); break; + case Builtin::BI__builtin_speculation_safe_value: + return SemaBuiltinSpeculationSafeValueOverloaded(TheCallResult); } // Since the target specific builtins for each arch overlap, only check those @@ -5308,6 +5310,50 @@ return TheCallResult; } +ExprResult +Sema::SemaBuiltinSpeculationSafeValueOverloaded(ExprResult TheCallResult) { + CallExpr *TheCall = (CallExpr *)TheCallResult.get(); + DeclRefExpr *DRE = + cast(TheCall->getCallee()->IgnoreParenCasts()); + FunctionDecl *FDecl = cast(DRE->getDecl()); + unsigned BuiltinID = FDecl->getBuiltinID(); + assert(BuiltinID == Builtin::BI__builtin_speculation_safe_value && + "Unexpected speculation_Safe_value builtin!"); + + // Too few args + if (TheCall->getNumArgs() < 1) + return Diag(TheCall->getEndLoc(), + diag::err_typecheck_call_too_few_args_at_least) + << 0 /*function call*/ << 1 /* min args */ << TheCall->getNumArgs(); + + // Too many args + if (TheCall->getNumArgs() > 1) + return Diag(TheCall->getEndLoc(), + diag::err_typecheck_call_too_many_args_at_most) + << 0 /*function call*/ << 1 << TheCall->getNumArgs() + << SourceRange(TheCall->getArg(1)->getBeginLoc(), + (*(TheCall->arg_end() - 1))->getEndLoc()); + + // Derive the return type from the pointer argument + ExprResult FirstArg = + DefaultFunctionArrayLvalueConversion(TheCall->getArg(0)); + if (FirstArg.isInvalid()) + return true; + TheCall->setArg(0, FirstArg.get()); + QualType FirstArgType = FirstArg.get()->getType(); + + TheCall->setType(FirstArgType); + + // The first argument must be a pointer or integer type. + if (!(FirstArgType->isIntegerType() || FirstArgType->isAnyPointerType())) + return Diag(TheCall->getArg(0)->getBeginLoc(), + diag::err_specsafevalue_builtin_must_be_pointer_or_integral) + << TheCall->getArg(0)->getType() + << TheCall->getArg(0)->getSourceRange(); + + return TheCallResult; +} + /// CheckObjCString - Checks that the argument to the builtin /// CFString constructor is correct /// Note: It might also make sense to do the UTF-16 conversion here (would diff --git a/clang/test/CodeGen/builtin-speculation-safe-value.c b/clang/test/CodeGen/builtin-speculation-safe-value.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/builtin-speculation-safe-value.c @@ -0,0 +1,18 @@ +// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm %s -o - | FileCheck -check-prefix=CHECK-SUPPORTED %s + +void test(char c, int i, void *p) { + // CHECK-LABEL-SUPPORTED: define void @test + + char c_safe = __builtin_speculation_safe_value(c); + // CHECK-SUPPORTED: call i8 @llvm.speculationsafevalue.i8(i8 %{{[0-9a-z]+}}) + + int i_safe = __builtin_speculation_safe_value(i); + // CHECK-SUPPORTED: call i32 @llvm.speculationsafevalue.i32(i32 %{{[0-9a-z]+}}) + + void *p_safe = __builtin_speculation_safe_value(p); + // CHECK-SUPPORTED: call i8* @llvm.speculationsafevalue.p0i8(i8* %{{[0-9a-z]+}}) + + int arr[4]; + int *arr_safe = __builtin_speculation_safe_value(arr); + // CHECK-SUPPORTED: call i32* @llvm.speculationsafevalue.p0i32(i32* %{{[0-9a-z]+}}) +} diff --git a/clang/test/Preprocessor/init.c b/clang/test/Preprocessor/init.c --- a/clang/test/Preprocessor/init.c +++ b/clang/test/Preprocessor/init.c @@ -9675,6 +9675,7 @@ // WEBASSEMBLY-NEXT:#define __GNUC_STDC_INLINE__ 1 // WEBASSEMBLY-NEXT:#define __GNUC__ {{.*}} // WEBASSEMBLY-NEXT:#define __GXX_ABI_VERSION 1002 +// WEBASSEMBLY-NEXT:#define __HAVE_SPECULATION_SAFE_VALUE 1 // WEBASSEMBLY32-NEXT:#define __ILP32__ 1 // WEBASSEMBLY64-NOT:#define __ILP32__ // WEBASSEMBLY-NEXT:#define __INT16_C_SUFFIX__ diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -16372,6 +16372,51 @@ On the other hand, if constant folding is not run, it will never evaluate to true, even in simple cases. +'``llvm.speculation_safe_value``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +This is an overloaded intrinsic. You can use llvm.speculation_safe_value on any +integer type, and any pointer type. However, not all targets support this +intrinsic at the moment. + +:: + + declare T @llvm.speculation_safe_value.T(T %val) + +Overview: +""""""""" + +The '``llvm.speculation_safe_value``' intrinsic. + +Arguments: +"""""""""" + +The first argument is a pointer or integer value. + + +Semantics: +"""""""""" + +On a processor that predicts the direction and target of branches, code executes +speculatively, i.e. before it is known if the code actually should be executed +according to program logic. + +When the processor executes code speculatively that it should not execute +according to program logic, the code is said to be executing on a +miss-speculated path. Miss-speculated paths are caused by incorrect prediction +of the direction or targets of branches. + +This intrinsic guarantees that for miss-speculated paths where at least the +direction of one of the previously executed conditional branches was +mispredicted, the intrinsic returns 0. + +On fully correctly predicted execution paths, it returns %val. + +For paths not covered by the above statements, it returns either 0 or %val. + Stack Map Intrinsics -------------------- diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -881,6 +881,8 @@ VECREDUCE_AND, VECREDUCE_OR, VECREDUCE_XOR, VECREDUCE_SMAX, VECREDUCE_SMIN, VECREDUCE_UMAX, VECREDUCE_UMIN, + SPECULATION_SAFE_VALUE, + /// BUILTIN_OP_END - This must be the last enum value in this list. /// The target-specific pre-isel opcode values start here. BUILTIN_OP_END diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1168,6 +1168,10 @@ def int_ssa_copy : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem, Returned<0>]>; + +//===----- Intrinsics to mitigate against miss-speculation exploits -------===// + +def int_speculationsafevalue : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], []>; //===----------------------------------------------------------------------===// // Target-specific intrinsics //===----------------------------------------------------------------------===// diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -280,6 +280,10 @@ SDTCisVT<2, OtherVT>, SDTCisVT<3, OtherVT>, SDTCisPtrTy<4>, SDTCisPtrTy<5> ]>; +def SDTSpeculationSafe: SDTypeProfile<1, 1, [ + SDTCisInt<1>, SDTCisSameAs<1, 0> +]>; + class SDCallSeqStart constraints> : SDTypeProfile<0, 2, constraints>; class SDCallSeqEnd constraints> : @@ -581,6 +585,8 @@ def assertsext : SDNode<"ISD::AssertSext", SDT_assertext>; def assertzext : SDNode<"ISD::AssertZext", SDT_assertext>; +def speculationsafevalue : SDNode<"ISD::SPECULATION_SAFE_VALUE", + SDTSpeculationSafe, []>; //===----------------------------------------------------------------------===// // Selection DAG Condition Codes diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -185,6 +185,9 @@ case ISD::VECREDUCE_UMIN: Res = PromoteIntRes_VECREDUCE(N); break; + case ISD::SPECULATION_SAFE_VALUE: + Res = PromoteIntRes_SpeculationSafeValue(N); + break; } // If the result is null then the sub-method took care of registering it. @@ -787,6 +790,13 @@ LHS.getValueType(), LHS, RHS); } +SDValue DAGTypeLegalizer::PromoteIntRes_SpeculationSafeValue(SDNode *N) { + // Propagate size promotion through the intrinsic. + SDValue Op = GetPromotedInteger(N->getOperand(0)); + return DAG.getNode(N->getOpcode(), SDLoc(N), + Op.getValueType(), Op); +} + SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { // Sign extend the input. SDValue LHS = SExtPromotedInteger(N->getOperand(0)); @@ -1680,6 +1690,9 @@ case ISD::UMULO: case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; + case ISD::SPECULATION_SAFE_VALUE: + ExpandIntRes_SPECULATION_SAFE_VALUE(N, Lo, Hi); break; + case ISD::SADDSAT: case ISD::UADDSAT: case ISD::SSUBSAT: @@ -2399,6 +2412,15 @@ Hi = DAG.getConstant(0, dl, NVT); } +void DAGTypeLegalizer::ExpandIntRes_SPECULATION_SAFE_VALUE(SDNode *N, + SDValue &Lo, + SDValue &Hi) { + SDLoc dl(N); + GetExpandedInteger(N->getOperand(0), Lo, Hi); + Lo = DAG.getNode(N->getOpcode(), dl, Lo.getValueType(), Lo); + Hi = DAG.getNode(N->getOpcode(), dl, Hi.getValueType(), Hi); +} + void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, SDValue &Lo, SDValue &Hi) { SDLoc dl(N); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -343,6 +343,7 @@ SDValue PromoteIntRes_UNDEF(SDNode *N); SDValue PromoteIntRes_VAARG(SDNode *N); SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo); + SDValue PromoteIntRes_SpeculationSafeValue(SDNode *N); SDValue PromoteIntRes_ADDSUBSAT(SDNode *N); SDValue PromoteIntRes_MULFIX(SDNode *N); SDValue PromoteIntRes_FLT_ROUNDS(SDNode *N); @@ -438,6 +439,7 @@ void ExpandIntRes_SADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_UADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_XMULO (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandIntRes_SPECULATION_SAFE_VALUE(SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_ADDSUBSAT (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_MULFIX (SDNode *N, SDValue &Lo, SDValue &Hi); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -6093,6 +6093,12 @@ setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg)); return nullptr; } + case Intrinsic::speculationsafevalue: { + SDValue Arg = getValue(I.getArgOperand(0)); + EVT Ty = Arg.getValueType(); + setValue(&I, DAG.getNode(ISD::SPECULATION_SAFE_VALUE, sdl, Ty, Arg)); + return nullptr; + } case Intrinsic::fshl: case Intrinsic::fshr: { bool IsFSHL = Intrinsic == Intrinsic::fshl; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -79,6 +79,7 @@ #ifndef NDEBUG case ISD::DELETED_NODE: return "<>"; #endif + case ISD::SPECULATION_SAFE_VALUE: return "SpeculationSafeValue"; case ISD::PREFETCH: return "Prefetch"; case ISD::ATOMIC_FENCE: return "AtomicFence"; case ISD::ATOMIC_CMP_SWAP: return "AtomicCmpSwap"; diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -29,6 +29,8 @@ // Start the numbering where the builtin ops leave off. FIRST_NUMBER = ISD::BUILTIN_OP_END, + SpeculationSafeValue, + /// Bit scan forward. BSF, /// Bit scan reverse. @@ -1191,6 +1193,8 @@ LegalFPImmediates.push_back(Imm); } + SDValue LowerSPECULATION_SAFE_VALUE(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerCallResult(SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -193,6 +193,9 @@ setCondCodeAction(ISD::SETUNE, MVT::f64, Expand); setCondCodeAction(ISD::SETUNE, MVT::f80, Expand); + setOperationAction(ISD::SPECULATION_SAFE_VALUE, MVT::i32, Custom); + setOperationAction(ISD::SPECULATION_SAFE_VALUE, MVT::i64, Custom); + // Integer absolute. if (Subtarget.hasCMov()) { setOperationAction(ISD::ABS , MVT::i16 , Custom); @@ -4751,6 +4754,17 @@ } } +SDValue +X86TargetLowering::LowerSPECULATION_SAFE_VALUE(SDValue Op, + SelectionDAG &DAG) const { + + assert((Op.getValueType() == MVT::i64 || Op.getValueType() == MVT::i32) && + "Unexpected lowering"); + + SDLoc DL(Op); + return DAG.getNode(X86ISD::SpeculationSafeValue, DL, Op.getValueType(), + Op.getOperand(0)); +} bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, @@ -26734,6 +26748,8 @@ case ISD::GC_TRANSITION_START: return LowerGC_TRANSITION_START(Op, DAG); case ISD::GC_TRANSITION_END: return LowerGC_TRANSITION_END(Op, DAG); + case ISD::SPECULATION_SAFE_VALUE: + return LowerSPECULATION_SAFE_VALUE(Op, DAG); } } @@ -27933,6 +27949,8 @@ case X86ISD::NT_BRIND: return "X86ISD::NT_BRIND"; case X86ISD::UMWAIT: return "X86ISD::UMWAIT"; case X86ISD::TPAUSE: return "X86ISD::TPAUSE"; + case X86ISD::SpeculationSafeValue: + return "X86ISD::SpeculationSafeValue"; } return nullptr; } diff --git a/llvm/lib/Target/X86/X86InstrInfo.td b/llvm/lib/Target/X86/X86InstrInfo.td --- a/llvm/lib/Target/X86/X86InstrInfo.td +++ b/llvm/lib/Target/X86/X86InstrInfo.td @@ -298,6 +298,9 @@ SDTCisVT<2, i32>, SDTCisVT<3, i32>]>, [SDNPHasChain, SDNPSideEffect]>; +def X86SpeculationSafeValue : SDNode<"X86ISD::SpeculationSafeValue", SDTIntUnaryOp>; + + //===----------------------------------------------------------------------===// // X86 Operand Definitions. // @@ -1164,6 +1167,15 @@ // Miscellaneous Instructions. // +let hasSideEffects = 1, isCodeGenOnly = 1 in { + def SpeculationSafeValue64 + : PseudoI<(outs GR64:$dst), (ins GR64:$src), + [(set GR64:$dst, (X86SpeculationSafeValue GR64:$src))]>; + def SpeculationSafeValue32 + : PseudoI<(outs GR32:$dst), (ins GR32:$src), + [(set GR32:$dst, (X86SpeculationSafeValue GR32:$src))]>; +} + let isBarrier = 1, hasSideEffects = 1, usesCustomInserter = 1, SchedRW = [WriteSystem] in def Int_eh_sjlj_setup_dispatch diff --git a/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp b/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp --- a/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp +++ b/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp @@ -212,6 +212,7 @@ void hardenIndirectCallOrJumpInstr( MachineInstr &MI, SmallDenseMap &AddrRegToHardenedReg); + bool lowerIntrinsic(MachineFunction &MF); }; } // end anonymous namespace @@ -402,16 +403,20 @@ LLVM_DEBUG(dbgs() << "********** " << getPassName() << " : " << MF.getName() << " **********\n"); - // Only run if this pass is forced enabled or we detect the relevant function - // attribute requesting SLH. - if (!EnableSpeculativeLoadHardening && - !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening)) - return false; - Subtarget = &MF.getSubtarget(); MRI = &MF.getRegInfo(); TII = Subtarget->getInstrInfo(); TRI = Subtarget->getRegisterInfo(); + bool Modified = lowerIntrinsic(MF); + + // Only run this pass completely if it is forced enabled or if we detect the + // relevant function attribute requesting SLH. Otherwise we should only check + // for intrinsics that we must lower by adding an lfence. + if (!EnableSpeculativeLoadHardening && + !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening)) { + + return false || Modified; + } // FIXME: Support for 32-bit. PS.emplace(MF, &X86::GR64_NOSPRegClass); @@ -597,6 +602,31 @@ } } +bool X86SpeculativeLoadHardeningPass::lowerIntrinsic(MachineFunction &MF) { + bool Modified = false; + for (MachineBasicBlock &MBB : MF) { + MachineBasicBlock::iterator MBBI = MBB.begin(); + MachineBasicBlock::iterator MBBE = MBB.end(); + while (MBBI != MBBE) { + MachineBasicBlock::iterator NMBBI = std::next(MBBI); + MachineInstr &MI = *MBBI; + unsigned Opcode = MI.getOpcode(); + if (Opcode == X86::SpeculationSafeValue64 || + Opcode == X86::SpeculationSafeValue32) { + BuildMI(MBB, NMBBI, DebugLoc(), TII->get(X86::LFENCE)); + ++NumInstsInserted; + ++NumLFENCEsInserted; + MRI->replaceRegWith(MI.getOperand(0).getReg(), + MI.getOperand(1).getReg()); + MI.eraseFromParent(); + Modified = true; + } + MBBI = NMBBI; + } + } + return Modified; +} + SmallVector X86SpeculativeLoadHardeningPass::collectBlockCondInfo(MachineFunction &MF) { SmallVector Infos; diff --git a/llvm/test/CodeGen/X86/speculative-load-hardening-intrinsic.ll b/llvm/test/CodeGen/X86/speculative-load-hardening-intrinsic.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/speculative-load-hardening-intrinsic.ll @@ -0,0 +1,71 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefix=X64 + +; ModuleID = 'hello.cpp' +source_filename = "hello.cpp" +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: noinline nounwind optnone uwtable +define dso_local i32 @_Z5foo32i(i32 %a) #0 { +entry: + %a.addr = alloca i32, align 4 + %b = alloca i32, align 4 + %b_safe = alloca i32, align 4 + %c = alloca i32, align 4 + store i32 %a, i32* %a.addr, align 4 + %0 = load i32, i32* %a.addr, align 4 + %mul = mul nsw i32 %0, 100 + store i32 %mul, i32* %b, align 4 + %1 = load i32, i32* %b, align 4 + %2 = call i32 @llvm.speculationsafevalue.i32(i32 %1) +; X64: movl -12(%rbp), %eax +; X64: lfence +; X64: movl %eax, -8(%rbp) + store i32 %2, i32* %b_safe, align 4 + %3 = load i32, i32* %b_safe, align 4 + %add = add nsw i32 %3, 100 + store i32 %add, i32* %c, align 4 + %4 = load i32, i32* %c, align 4 + ret i32 %4 +} + +; Function Attrs: nounwind +declare i32 @llvm.speculationsafevalue.i32(i32) #1 + +; Function Attrs: noinline nounwind optnone uwtable +define dso_local i32 @_Z5foo64i(i32 %a) #0 { +entry: + %a.addr = alloca i32, align 4 + %b = alloca i64, align 8 + %b_safe = alloca i64, align 8 + %c = alloca i64, align 8 + store i32 %a, i32* %a.addr, align 4 + %0 = load i32, i32* %a.addr, align 4 + %mul = mul nsw i32 %0, 100 + %conv = sext i32 %mul to i64 + store i64 %conv, i64* %b, align 8 + %1 = load i64, i64* %b, align 8 + %2 = call i64 @llvm.speculationsafevalue.i64(i64 %1) +; X64: movq -32(%rbp), %rax +; X64: lfence +; X64: movq %rax, -24(%rbp) + store i64 %2, i64* %b_safe, align 8 + %3 = load i64, i64* %b_safe, align 8 + %add = add nsw i64 %3, 100 + store i64 %add, i64* %c, align 8 + %4 = load i64, i64* %c, align 8 + %conv1 = trunc i64 %4 to i32 + ret i32 %conv1 +} + +; Function Attrs: nounwind +declare i64 @llvm.speculationsafevalue.i64(i64) #1 + +attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nounwind } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 9.0.0 (https://github.com/llvm/llvm-project.git 6fd90b5505fe7cddd0fd798fe9608ea0e0325302)"}