Index: include/llvm/IR/CallingConv.h =================================================================== --- include/llvm/IR/CallingConv.h +++ include/llvm/IR/CallingConv.h @@ -147,7 +147,13 @@ /// \brief MSVC calling convention that passes vectors and vector aggregates /// in SSE registers. - X86_VectorCall = 80 + X86_VectorCall = 80, + + /// X86_INTR - x86 hardware interrupt context. Callee may take one + /// parameter representing the hardware error code, the presence of which + /// depends on the interrupt vector taken. Valid for both 32- and 64-bit + /// subtargets. + X86_INTR = 81 }; } // End CallingConv namespace Index: lib/AsmParser/LLLexer.cpp =================================================================== --- lib/AsmParser/LLLexer.cpp +++ lib/AsmParser/LLLexer.cpp @@ -587,6 +587,7 @@ KEYWORD(preserve_mostcc); KEYWORD(preserve_allcc); KEYWORD(ghccc); + KEYWORD(x86_intrcc); KEYWORD(cc); KEYWORD(c); Index: lib/AsmParser/LLParser.cpp =================================================================== --- lib/AsmParser/LLParser.cpp +++ lib/AsmParser/LLParser.cpp @@ -1521,6 +1521,7 @@ /// ::= 'preserve_mostcc' /// ::= 'preserve_allcc' /// ::= 'ghccc' +/// ::= 'x86_intrcc' /// ::= 'cc' UINT /// bool LLParser::ParseOptionalCallingConv(unsigned &CC) { @@ -1549,6 +1550,7 @@ case lltok::kw_preserve_mostcc:CC = CallingConv::PreserveMost; break; case lltok::kw_preserve_allcc: CC = CallingConv::PreserveAll; break; case lltok::kw_ghccc: CC = CallingConv::GHC; break; + case lltok::kw_x86_intrcc: CC = CallingConv::X86_INTR; break; case lltok::kw_cc: { Lex.Lex(); return ParseUInt32(CC); Index: lib/AsmParser/LLToken.h =================================================================== --- lib/AsmParser/LLToken.h +++ lib/AsmParser/LLToken.h @@ -97,6 +97,7 @@ kw_webkit_jscc, kw_anyregcc, kw_preserve_mostcc, kw_preserve_allcc, kw_ghccc, + kw_x86_intrcc, // Attributes: kw_attributes, Index: lib/IR/AsmWriter.cpp =================================================================== --- lib/IR/AsmWriter.cpp +++ lib/IR/AsmWriter.cpp @@ -319,6 +319,7 @@ case CallingConv::X86_64_Win64: Out << "x86_64_win64cc"; break; case CallingConv::SPIR_FUNC: Out << "spir_func"; break; case CallingConv::SPIR_KERNEL: Out << "spir_kernel"; break; + case CallingConv::X86_INTR: Out << "x86_intrcc"; break; } } Index: lib/Target/X86/X86CallingConv.td =================================================================== --- lib/Target/X86/X86CallingConv.td +++ lib/Target/X86/X86CallingConv.td @@ -708,6 +708,12 @@ CCDelegateTo ]>; +def CC_X86_Intr : CallingConv<[ + // Error codes aren't necessarily 32 bits wide, but fill a 32-bit slot. + CCPromoteToType, + CCAssignToStack<4, 8> +]>; + //===----------------------------------------------------------------------===// // X86 Root Argument Calling Conventions //===----------------------------------------------------------------------===// @@ -745,6 +751,7 @@ // This is the argument convention used for the entire X86 backend. def CC_X86 : CallingConv<[ CCIfCC<"CallingConv::Intel_OCL_BI", CCDelegateTo>, + CCIfCC<"CallingConv::X86_INTR", CCDelegateTo>, CCIfSubtarget<"is64Bit()", CCDelegateTo>, CCDelegateTo ]>; @@ -778,6 +785,12 @@ R11, R12, R13, R14, R15, RBP, (sequence "XMM%u", 0, 15))>; +def CSR_32_AllRegs : CalleeSavedRegs<(add EAX, EBX, ECX, EDX, EBP, ESI, + EDI, ESP, + (sequence "ST%u", 0, 7))>; +def CSR_32_AllRegs_SSE : CalleeSavedRegs<(add CSR_32_AllRegs, + (sequence "XMM%u", 0, 7))>; + def CSR_64_AllRegs : CalleeSavedRegs<(add CSR_64_MostRegs, RAX, RSP, (sequence "XMM%u", 16, 31))>; def CSR_64_AllRegs_AVX : CalleeSavedRegs<(sub (add CSR_64_MostRegs, RAX, RSP, Index: lib/Target/X86/X86ExpandPseudo.cpp =================================================================== --- lib/Target/X86/X86ExpandPseudo.cpp +++ lib/Target/X86/X86ExpandPseudo.cpp @@ -141,6 +141,15 @@ // The EH_RETURN pseudo is really removed during the MC Lowering. return true; } + case X86::IRET: { + // Adjust stack to erase error code + int64_t StackAdj = MBBI->getOperand(0).getImm(); + X86FL->emitSPUpdate(MBB, MBBI, StackAdj, true); + // Replace pseudo with machine iret + BuildMI(MBB, MBBI, DL, TII->get(STI->is64Bit() ? X86::IRET64 : X86::IRET32)); + MBB.erase(MBBI); + return true; + } } llvm_unreachable("Previous switch has a fallthrough?"); } Index: lib/Target/X86/X86ISelLowering.h =================================================================== --- lib/Target/X86/X86ISelLowering.h +++ lib/Target/X86/X86ISelLowering.h @@ -126,6 +126,9 @@ /// 1 is the number of bytes of stack to pop. RET_FLAG, + /// Return from interrupt. Operand 0 is the number of bytes to pop. + IRET, + /// Repeat fill, corresponds to X86::REP_STOSx. REP_STOS, Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -2057,6 +2057,9 @@ MachineFunction &MF = DAG.getMachineFunction(); X86MachineFunctionInfo *FuncInfo = MF.getInfo(); + if (CallConv == CallingConv::X86_INTR && !Outs.empty()) + report_fatal_error("X86 interrupts may not return any value"); + SmallVector RVLocs; CCState CCInfo(CallConv, isVarArg, MF, RVLocs, *DAG.getContext()); CCInfo.AnalyzeReturn(Outs, RetCC_X86); @@ -2170,7 +2173,10 @@ if (Flag.getNode()) RetOps.push_back(Flag); - return DAG.getNode(X86ISD::RET_FLAG, dl, MVT::Other, RetOps); + X86ISD::NodeType opcode = X86ISD::RET_FLAG; + if (CallConv == CallingConv::X86_INTR) + opcode = X86ISD::IRET; + return DAG.getNode(opcode, dl, MVT::Other, RetOps); } bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const { @@ -2412,6 +2418,14 @@ } else { int FI = MFI->CreateFixedObject(ValVT.getSizeInBits()/8, VA.getLocMemOffset(), isImmutable); + // Adjust SP offset of interrupt parameter, filling the slot + // normally taken by a return address. + if (CallConv == CallingConv::X86_INTR) { + const X86Subtarget& Subtarget = + static_cast(DAG.getSubtarget()); + MFI->setObjectOffset(FI, Subtarget.is64Bit() ? -8 : -4); + } + SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout())); SDValue Val = DAG.getLoad( ValVT, dl, Chain, FIN, @@ -2496,6 +2510,13 @@ assert(!(isVarArg && IsTailCallConvention(CallConv)) && "Var args not supported with calling convention fastcc, ghc or hipe"); + if (CallConv == CallingConv::X86_INTR) { + bool isLegal = Ins.size() == 0 + || (Ins.size() == 1 && Ins[0].VT == MVT::i32); + if (!isLegal) + report_fatal_error("X86 interrupts may take only one i32 argument"); + } + // Assign locations to all of the incoming arguments. SmallVector ArgLocs; CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); @@ -2782,6 +2803,9 @@ if (X86::isCalleePop(CallConv, Is64Bit, isVarArg, MF.getTarget().Options.GuaranteedTailCallOpt)) { FuncInfo->setBytesToPopOnReturn(StackSize); // Callee pops everything. + } else if (CallConv == CallingConv::X86_INTR && Ins.size() != 0) { + // Interrupts must pop the error code if present + FuncInfo->setBytesToPopOnReturn(Is64Bit ? 8 : 4); } else { FuncInfo->setBytesToPopOnReturn(0); // Callee pops nothing. // If this is an sret function, the return should pop the hidden pointer. @@ -2914,6 +2938,9 @@ X86MachineFunctionInfo *X86Info = MF.getInfo(); auto Attr = MF.getFunction()->getFnAttribute("disable-tail-calls"); + if (CallConv == CallingConv::X86_INTR) + report_fatal_error("X86 interrupts may not be called directly"); + if (Attr.getValueAsString() == "true") isTailCall = false; @@ -19221,6 +19248,7 @@ case X86ISD::CMOV: return "X86ISD::CMOV"; case X86ISD::BRCOND: return "X86ISD::BRCOND"; case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; + case X86ISD::IRET: return "X86ISD::IRET"; case X86ISD::REP_STOS: return "X86ISD::REP_STOS"; case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS"; case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg"; Index: lib/Target/X86/X86InstrControl.td =================================================================== --- lib/Target/X86/X86InstrControl.td +++ lib/Target/X86/X86InstrControl.td @@ -53,6 +53,19 @@ "{l}ret{|f}q\t$amt", [], IIC_RET>, Requires<[In64BitMode]>; def LRETIW : Ii16<0xCA, RawFrm, (outs), (ins i16imm:$amt), "{l}ret{w|f}\t$amt", [], IIC_RET>, OpSize16; + + // The machine return from interrupt instruction, but sometimes we need to + // perform a post-epilogue stack adjustment. Codegen emits the pseudo form + // which expands to include an SP adjustment if necessary. + def IRET16 : I <0xcf, RawFrm, (outs), (ins), "iret{w}", [], IIC_IRET>, + OpSize16; + def IRET32 : I <0xcf, RawFrm, (outs), (ins), "iret{l|d}", [], + IIC_IRET>, OpSize32; + def IRET64 : RI <0xcf, RawFrm, (outs), (ins), "iretq", [], + IIC_IRET>, Requires<[In64BitMode]>; + let isCodeGenOnly = 1 in + def IRET : PseudoI<(outs), (ins i16imm:$adj), [(X86iret timm:$adj)]>; + } // Unconditional branches. Index: lib/Target/X86/X86InstrInfo.td =================================================================== --- lib/Target/X86/X86InstrInfo.td +++ lib/Target/X86/X86InstrInfo.td @@ -156,6 +156,8 @@ def X86retflag : SDNode<"X86ISD::RET_FLAG", SDTX86Ret, [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>; +def X86iret : SDNode<"X86ISD::IRET", SDTX86Ret, + [SDNPHasChain, SDNPOptInGlue]>; def X86vastart_save_xmm_regs : SDNode<"X86ISD::VASTART_SAVE_XMM_REGS", Index: lib/Target/X86/X86InstrSystem.td =================================================================== --- lib/Target/X86/X86InstrSystem.td +++ lib/Target/X86/X86InstrSystem.td @@ -60,12 +60,6 @@ IIC_SYS_ENTER_EXIT>, TB; def SYSEXIT64 :RI<0x35, RawFrm, (outs), (ins), "sysexit{q}", [], IIC_SYS_ENTER_EXIT>, TB, Requires<[In64BitMode]>; - -def IRET16 : I<0xcf, RawFrm, (outs), (ins), "iret{w}", [], IIC_IRET>, OpSize16; -def IRET32 : I<0xcf, RawFrm, (outs), (ins), "iret{l|d}", [], IIC_IRET>, - OpSize32; -def IRET64 : RI<0xcf, RawFrm, (outs), (ins), "iretq", [], IIC_IRET>, - Requires<[In64BitMode]>; } // SchedRW def : Pat<(debugtrap), Index: lib/Target/X86/X86RegisterInfo.cpp =================================================================== --- lib/Target/X86/X86RegisterInfo.cpp +++ lib/Target/X86/X86RegisterInfo.cpp @@ -216,6 +216,7 @@ const MCPhysReg * X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const { const X86Subtarget &Subtarget = MF->getSubtarget(); + bool HasSSE = Subtarget.hasSSE1(); bool HasAVX = Subtarget.hasAVX(); bool HasAVX512 = Subtarget.hasAVX512(); bool CallsEHReturn = MF->getMMI().callsEHReturn(); @@ -258,6 +259,18 @@ if (CallsEHReturn) return CSR_64EHRet_SaveList; return CSR_64_SaveList; + case CallingConv::X86_INTR: + if (Is64Bit) { + if (HasAVX) + return CSR_64_AllRegs_AVX_SaveList; + else + return CSR_64_AllRegs_SaveList; + } else { + if (HasSSE) + return CSR_32_AllRegs_SSE_SaveList; + else + return CSR_32_AllRegs_SaveList; + } default: break; } @@ -278,6 +291,7 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF, CallingConv::ID CC) const { const X86Subtarget &Subtarget = MF.getSubtarget(); + bool HasSSE = Subtarget.hasSSE1(); bool HasAVX = Subtarget.hasAVX(); bool HasAVX512 = Subtarget.hasAVX512(); @@ -312,12 +326,24 @@ if (Is64Bit) return CSR_64_MostRegs_RegMask; break; - default: - break; case CallingConv::X86_64_Win64: return CSR_Win64_RegMask; case CallingConv::X86_64_SysV: return CSR_64_RegMask; + case CallingConv::X86_INTR: + if (Is64Bit) { + if (HasAVX) + return CSR_64_AllRegs_AVX_RegMask; + else + return CSR_64_AllRegs_RegMask; + } else { + if (HasSSE) + return CSR_32_AllRegs_SSE_RegMask; + else + return CSR_32_AllRegs_RegMask; + } + default: + break; } // Unlike getCalleeSavedRegs(), we don't have MMI so we can't check Index: test/CodeGen/X86/x86-intrcc.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/x86-intrcc.ll @@ -0,0 +1,61 @@ +; RUN: llc -mtriple=x86_64-unknown-unknown < %s | FileCheck %s -check-prefix=X64 +; RUN: llc -mtriple=i686-unknown-unknown < %s | FileCheck %s -check-prefix=X32 + +; No stack adjustment if declared with no parameters +define x86_intrcc void @test_isr_noparams() { + ; CHECK-LABEL: test_isr_noparams: + ; X32: iretl + ; X64: iretq + ret void +} + +; Spills rax, putting original rsp at +8. Stack is adjusted up another 8 bytes +; before return, popping the error code. +define x86_intrcc void @test_isr_oneparam(i32 %ecode) { + ; CHECK-LABEL: test_isr_oneparam + ; X64: pushq %rax + ; X64: movl 8(%rsp), %eax + ; X64: popq %rax + ; X64: addq $8, %rsp + ; X64: iretq + ; X32: pushl %eax + ; X32: movl 4(%ebp), %eax + ; X32: popl %eax + ; X32: addl $4, %esp + ; X32: iretl + call void asm sideeffect "movl $0, %eax", "m,~{eax}"(i32 %ecode) + ret void +} + +; All clobbered registers must be saved +define x86_intrcc void @test_isr_clobbers64(i32 %ecode) { + call void asm sideeffect "", "~{rax},~{rbx},~{rbp},~{r11},~{xmm0}"() + ; LABEL: test_isr_clobbers64 + ; X64-SSE-NEXT: movaps %xmm0 + ; X64-SSE-NEXT: pushq %rbx + ; X64-SSE-NEXT: pushq %rbp + ; X64-SSE-NEXT; pushq %r11 + ; X64-SSE-NEXT: pushq %rax + ; X64-SSE-NEXT: popq %rax + ; X64-SSE-NEXT: popq %r11 + ; X64-SSE-NEXT: popq %rbp + ; X64-SSE-NEXT: popq %rbx + ; X64-SSE-NEXT: movaps %xmm0 + ; X64: addq $8, %rsp + ; X64-NEXT: iretq + ret void +} + +define x86_intrcc void @test_isr_clobbers32(i32 %ecode) { + call void asm sideeffect "", "~{eax},~{ebx},~{ebp}"() + ; LABEL: test_isr_clobbers32 + ; X32: pushl %ebp + ; X32: pushl %ebx + ; X32; pushl %eax + ; X32: popl %eax + ; X32: popl %ebx + ; X32: popl %ebp + ; X32: addl $4, %esp + ; X32-NEXT: iretl + ret void +}