Index: llvm/lib/Target/X86/X86ExpandPseudo.cpp =================================================================== --- llvm/lib/Target/X86/X86ExpandPseudo.cpp +++ llvm/lib/Target/X86/X86ExpandPseudo.cpp @@ -371,6 +371,31 @@ MBBI->eraseFromParent(); return true; } + case X86::MWAITX_SAVE_EBX: + case X86::MWAITX_SAVE_RBX: + { + // Perform the following transformation. + // SaveRbx = pseudomwaitx InArg, SaveRbx + // => + // [E|R]BX = InArg + // actualmwaitx + // [E|R]BX = SaveRbx + const MachineOperand &InArg = MBBI->getOperand(1); + // Copy the input argument of the pseudo into the argument of the + // actual instruction. + TII->copyPhysReg(MBB, MBBI, DL, X86::EBX, InArg.getReg(), + InArg.isKill()); + // Create the actual instruction. + BuildMI(MBB, MBBI, DL, TII->get(X86::MWAITXrrr)); + // Finally, restore the value of RBX. + Register SaveRbx = MBBI->getOperand(2).getReg(); + unsigned BasePointer = Opcode == X86::MWAITX_SAVE_EBX ? X86::EBX : X86::RBX; + TII->copyPhysReg(MBB, MBBI, DL, BasePointer, SaveRbx, + /*SrcIsKill*/ true); + // Delete the pseudo. + MBBI->eraseFromParent(); + return true; + } case TargetOpcode::ICALL_BRANCH_FUNNEL: ExpandICallBranchFunnel(&MBB, MBBI); return true; Index: llvm/lib/Target/X86/X86ISelLowering.h =================================================================== --- llvm/lib/Target/X86/X86ISelLowering.h +++ llvm/lib/Target/X86/X86ISelLowering.h @@ -626,6 +626,10 @@ // Vector signed/unsigned integer to float/double. STRICT_CVTSI2P, STRICT_CVTUI2P, + // Mwaitx builtin is lowered to this if the base pointer needs saving. + MWAITX_SAVE_EBX_DAG, + MWAITX_SAVE_RBX_DAG, + // Compare and swap. LCMPXCHG_DAG = ISD::FIRST_TARGET_MEMORY_OPCODE, LCMPXCHG8_DAG, Index: llvm/lib/Target/X86/X86ISelLowering.cpp =================================================================== --- llvm/lib/Target/X86/X86ISelLowering.cpp +++ llvm/lib/Target/X86/X86ISelLowering.cpp @@ -24754,6 +24754,44 @@ return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC, Operation.getValue(1)); } + case Intrinsic::x86_mwaitx: + { + // If the current function needs the base pointer, RBX, + // we shouldn't use mwaitx directly. + // Indeed the lowering of that instruction will clobber + // that register and since RBX will be a reserved register + // the register allocator will not make sure its value will + // be properly saved and restored around this live-range. + SDLoc dl(Op); + const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); + Register BasePtr = TRI->getBaseRegister(); + if (!TRI->hasBasePointer(DAG.getMachineFunction())) + break; + if (!(BasePtr == X86::RBX || BasePtr == X86::EBX)) + break; + + // Copy first 2 arguments to ECX and EAX but not EBX just yet. + SDValue Chain = DAG.getCopyToReg(Op->getOperand(0), dl, X86::ECX, + Op->getOperand(2), SDValue()); + SDValue InFlag = Chain.getValue(1); + + Chain = DAG.getCopyToReg(Chain, dl, X86::EAX, Op->getOperand(3), InFlag); + InFlag = Chain.getValue(1); + + // Save base pointer which is either EBX or RBX. + EVT Ty = BasePtr == X86::RBX ? MVT::i64 : MVT::i32; + Chain = DAG.getCopyFromReg(Op->getOperand(0), dl, BasePtr, Ty, InFlag); + InFlag = Chain.getValue(2); + + unsigned Opcode = BasePtr == X86::RBX ? X86ISD::MWAITX_SAVE_RBX_DAG + : X86ISD::MWAITX_SAVE_EBX_DAG; + Chain = DAG.getNode(Opcode, dl, {MVT::Other, MVT::Glue}, + {/*Chain*/ Chain.getValue(1), + /*ebx input*/ Op->getOperand(4), + /*BP save*/ Chain, /*Glue*/ InFlag}); + + return Chain; + } } return SDValue(); } @@ -29535,6 +29573,10 @@ return "X86ISD::LCMPXCHG8_SAVE_EBX_DAG"; case X86ISD::LCMPXCHG16_SAVE_RBX_DAG: return "X86ISD::LCMPXCHG16_SAVE_RBX_DAG"; + case X86ISD::MWAITX_SAVE_EBX_DAG: + return "X86ISD::MWAITX_SAVE_EBX_DAG"; + case X86ISD::MWAITX_SAVE_RBX_DAG: + return "X86ISD::MWAITX_SAVE_RBX_DAG"; case X86ISD::LADD: return "X86ISD::LADD"; case X86ISD::LSUB: return "X86ISD::LSUB"; case X86ISD::LOR: return "X86ISD::LOR"; @@ -32321,6 +32363,14 @@ BB->addLiveIn(BasePtr); return BB; } + case X86::MWAITX_SAVE_EBX: + case X86::MWAITX_SAVE_RBX: { + unsigned BasePtr = + MI.getOpcode() == X86::MWAITX_SAVE_EBX ? X86::EBX : X86::RBX; + if (!BB->isLiveIn(BasePtr)) + BB->addLiveIn(BasePtr); + return BB; + } } } Index: llvm/lib/Target/X86/X86InstrCompiler.td =================================================================== --- llvm/lib/Target/X86/X86InstrCompiler.td +++ llvm/lib/Target/X86/X86InstrCompiler.td @@ -925,6 +925,34 @@ GR64:$rbx_save))]>; } +// This pseudo must be used when the frame uses RBX as +// the base pointer. +// cf comment for LCMPXCHG8B_SAVE_EBX. +let Defs = [ECX, EAX, EBX, EFLAGS], Uses = [ECX, EAX, EBX], + Predicates = [HasMWAITX], SchedRW = [WriteSystem], + isCodeGenOnly = 1, isPseudo = 1, Constraints = "$ebx_save = $dst", + usesCustomInserter = 1 in { +def MWAITX_SAVE_EBX : + I<0, Pseudo, (outs GR32:$dst), + (ins GR32:$ebx_input, GR32:$ebx_save), + "mwaitx", + [(set GR32:$dst, (X86mwaitsave_ebx GR32:$ebx_input, + GR32:$ebx_save))]>; +} +// Same as MWAITX_SAVE_EBX but for the case where RBX is the base pointer. +let Defs = [ECX, EAX, EBX, EFLAGS], Uses = [ECX, EAX, EBX], + Predicates = [HasMWAITX], SchedRW = [WriteSystem], + isCodeGenOnly = 1, isPseudo = 1, Constraints = "$rbx_save = $dst", + usesCustomInserter = 1 in { +def MWAITX_SAVE_RBX : + I<0, Pseudo, (outs GR64:$dst), + (ins GR32:$ebx_input, GR64:$rbx_save), + "mwaitx", + [(set GR64:$dst, (X86mwaitsave_rbx GR32:$ebx_input, + GR64:$rbx_save))]>; +} + + defm LCMPXCHG : LCMPXCHG_BinOp<0xB0, 0xB1, MRMDestMem, "cmpxchg", X86cas>; // Atomic exchange and add Index: llvm/lib/Target/X86/X86InstrInfo.td =================================================================== --- llvm/lib/Target/X86/X86InstrInfo.td +++ llvm/lib/Target/X86/X86InstrInfo.td @@ -77,6 +77,11 @@ [SDTCisVT<0, i64>, SDTCisPtrTy<1>, SDTCisVT<2, i64>, SDTCisVT<3, i64>]>; +def SDTX86mwaitxpairSaveEbx : SDTypeProfile<1, 2, [SDTCisVT<0, i32>, + SDTCisVT<1, i32>, SDTCisVT<2, i32>]>; +def SDTX86mwaitxpairSaveRbx : SDTypeProfile<1, 2, [SDTCisVT<0, i64>, + SDTCisVT<1, i32>, SDTCisVT<2, i64>]>; + def SDTLockBinaryArithWithFlags : SDTypeProfile<1, 2, [SDTCisVT<0, i32>, SDTCisPtrTy<1>, SDTCisInt<2>]>; @@ -183,6 +188,15 @@ [SDNPHasChain, SDNPInGlue, SDNPOutGlue, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>; +def X86mwaitsave_ebx : SDNode<"X86ISD::MWAITX_SAVE_EBX_DAG", + SDTX86mwaitxpairSaveEbx, + [SDNPHasChain, SDNPInGlue, SDNPOutGlue, + SDNPMayStore, SDNPMayLoad]>; +def X86mwaitsave_rbx : SDNode<"X86ISD::MWAITX_SAVE_RBX_DAG", + SDTX86mwaitxpairSaveRbx, + [SDNPHasChain, SDNPInGlue, SDNPOutGlue, + SDNPMayStore, SDNPMayLoad]>; + def X86retflag : SDNode<"X86ISD::RET_FLAG", SDTX86Ret, [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>; def X86iret : SDNode<"X86ISD::IRET", SDTX86Ret, Index: llvm/test/CodeGen/X86/base-pointer-and-mwaitx.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/base-pointer-and-mwaitx.ll @@ -0,0 +1,46 @@ +; RUN: llc -mtriple=x86_64-pc-linux-gnu -mattr=+mwaitx -x86-use-base-pointer=true -stackrealign -stack-alignment=32 %s -o - | FileCheck --check-prefix=CHECK --check-prefix=USE_BASE --check-prefix=USE_BASE_64 %s +; RUN: llc -mtriple=x86_64-pc-linux-gnux32 -mattr=+mwaitx -x86-use-base-pointer=true -stackrealign -stack-alignment=32 %s -o - | FileCheck --check-prefix=CHECK --check-prefix=USE_BASE --check-prefix=USE_BASE_32 %s + +; This test checks that we save and restore the base pointer (ebx or rbx) in the +; presence of the mwaitx intrinsic which requires to use ebx for one of its +; argument. +; This function uses a dynamically allocated stack to force the use +; of a base pointer. +; After the call to the mwaitx intrinsic we do a volatile store to the +; dynamically allocated memory which will require the use of the base pointer. +; The base pointer should therefore be restored straight after the mwaitx +; instruction. + +; CHECK-LABEL: test_baseptr: +; USE_BASE_64: movq %rsp, %rbx +; Save base pointer. +; USE_BASE_64: movq %rbx, [[SAVE_rbx:%r([8-9]|1[0-5]|di|si)]] +; Set mwaitx ebx argument. +; USE_BASE_64: movl {{[^ ]+}}, %ebx +; USE_BASE_64-NEXT: mwaitx +; Restore base pointer. +; USE_BASE_64-NEXT: movq [[SAVE_rbx]], %rbx + +; USE_BASE_32: movl %esp, %ebx +; Save base pointer. +; USE_BASE_32: movl %ebx, [[SAVE_ebx:%e(di|si)]] +; Set mwaitx ebx argument. +; USE_BASE_32: movl {{[^ ]+}}, %ebx +; USE_BASE_32-NEXT: mwaitx +; Restore base pointer. +; USE_BASE_32-NEXT: movl [[SAVE_ebx]], %ebx + + +define void @test_baseptr(i64 %x, i64 %y, i32 %E, i32 %H, i32 %C) nounwind { +entry: + %ptr = alloca i8*, align 8 + %0 = alloca i8, i64 %x, align 16 + store i8* %0, i8** %ptr, align 8 + call void @llvm.x86.mwaitx(i32 %E, i32 %H, i32 %C) + %1 = load i8*, i8** %ptr, align 8 + %arrayidx = getelementptr inbounds i8, i8* %1, i64 %y + store volatile i8 42, i8* %arrayidx, align 1 + ret void +} + +declare void @llvm.x86.mwaitx(i32, i32, i32) nounwind