Index: include/llvm/IR/CallingConv.h
===================================================================
--- include/llvm/IR/CallingConv.h
+++ include/llvm/IR/CallingConv.h
@@ -147,7 +147,13 @@
/// \brief MSVC calling convention that passes vectors and vector aggregates
/// in SSE registers.
- X86_VectorCall = 80
+ X86_VectorCall = 80,
+
+ /// X86_INTR - x86 hardware interrupt context. Callee may take one
+ /// parameter representing the hardware error code, the presence of which
+ /// depends on the interrupt vector taken. Valid for both 32- and 64-bit
+ /// subtargets.
+ X86_INTR = 81
};
} // End CallingConv namespace
Index: lib/AsmParser/LLLexer.cpp
===================================================================
--- lib/AsmParser/LLLexer.cpp
+++ lib/AsmParser/LLLexer.cpp
@@ -587,6 +587,7 @@
KEYWORD(preserve_mostcc);
KEYWORD(preserve_allcc);
KEYWORD(ghccc);
+ KEYWORD(x86_intrcc);
KEYWORD(cc);
KEYWORD(c);
Index: lib/AsmParser/LLParser.cpp
===================================================================
--- lib/AsmParser/LLParser.cpp
+++ lib/AsmParser/LLParser.cpp
@@ -1521,6 +1521,7 @@
/// ::= 'preserve_mostcc'
/// ::= 'preserve_allcc'
/// ::= 'ghccc'
+/// ::= 'x86_intrcc'
/// ::= 'cc' UINT
///
bool LLParser::ParseOptionalCallingConv(unsigned &CC) {
@@ -1549,6 +1550,7 @@
case lltok::kw_preserve_mostcc:CC = CallingConv::PreserveMost; break;
case lltok::kw_preserve_allcc: CC = CallingConv::PreserveAll; break;
case lltok::kw_ghccc: CC = CallingConv::GHC; break;
+ case lltok::kw_x86_intrcc: CC = CallingConv::X86_INTR; break;
case lltok::kw_cc: {
Lex.Lex();
return ParseUInt32(CC);
Index: lib/AsmParser/LLToken.h
===================================================================
--- lib/AsmParser/LLToken.h
+++ lib/AsmParser/LLToken.h
@@ -97,6 +97,7 @@
kw_webkit_jscc, kw_anyregcc,
kw_preserve_mostcc, kw_preserve_allcc,
kw_ghccc,
+ kw_x86_intrcc,
// Attributes:
kw_attributes,
Index: lib/IR/AsmWriter.cpp
===================================================================
--- lib/IR/AsmWriter.cpp
+++ lib/IR/AsmWriter.cpp
@@ -319,6 +319,7 @@
case CallingConv::X86_64_Win64: Out << "x86_64_win64cc"; break;
case CallingConv::SPIR_FUNC: Out << "spir_func"; break;
case CallingConv::SPIR_KERNEL: Out << "spir_kernel"; break;
+ case CallingConv::X86_INTR: Out << "x86_intrcc"; break;
}
}
Index: lib/Target/X86/X86CallingConv.td
===================================================================
--- lib/Target/X86/X86CallingConv.td
+++ lib/Target/X86/X86CallingConv.td
@@ -708,6 +708,12 @@
CCDelegateTo<CC_X86_32_C>
]>;
+def CC_X86_Intr : CallingConv<[
+ // Error codes aren't necessarily 32 bits wide, but fill a 32-bit slot.
+ CCPromoteToType<i32>,
+ CCAssignToStack<4, 8>
+]>;
+
//===----------------------------------------------------------------------===//
// X86 Root Argument Calling Conventions
//===----------------------------------------------------------------------===//
@@ -745,6 +751,7 @@
// This is the argument convention used for the entire X86 backend.
def CC_X86 : CallingConv<[
CCIfCC<"CallingConv::Intel_OCL_BI", CCDelegateTo<CC_Intel_OCL_BI>>,
+ CCIfCC<"CallingConv::X86_INTR", CCDelegateTo<CC_X86_Intr>>,
CCIfSubtarget<"is64Bit()", CCDelegateTo<CC_X86_64>>,
CCDelegateTo<CC_X86_32>
]>;
@@ -778,6 +785,12 @@
R11, R12, R13, R14, R15, RBP,
(sequence "XMM%u", 0, 15))>;
+def CSR_32_AllRegs : CalleeSavedRegs<(add EAX, EBX, ECX, EDX, EBP, ESI,
+ EDI, ESP,
+ (sequence "ST%u", 0, 7))>;
+def CSR_32_AllRegs_SSE : CalleeSavedRegs<(add CSR_32_AllRegs,
+ (sequence "XMM%u", 0, 7))>;
+
def CSR_64_AllRegs : CalleeSavedRegs<(add CSR_64_MostRegs, RAX, RSP,
(sequence "XMM%u", 16, 31))>;
def CSR_64_AllRegs_AVX : CalleeSavedRegs<(sub (add CSR_64_MostRegs, RAX, RSP,
Index: lib/Target/X86/X86ExpandPseudo.cpp
===================================================================
--- lib/Target/X86/X86ExpandPseudo.cpp
+++ lib/Target/X86/X86ExpandPseudo.cpp
@@ -141,6 +141,15 @@
// The EH_RETURN pseudo is really removed during the MC Lowering.
return true;
}
+ case X86::IRET: {
+ // Adjust stack to erase error code
+ int64_t StackAdj = MBBI->getOperand(0).getImm();
+ X86FL->emitSPUpdate(MBB, MBBI, StackAdj, true);
+ // Replace pseudo with machine iret
+ BuildMI(MBB, MBBI, DL, TII->get(STI->is64Bit() ? X86::IRET64 : X86::IRET32));
+ MBB.erase(MBBI);
+ return true;
+ }
}
llvm_unreachable("Previous switch has a fallthrough?");
}
Index: lib/Target/X86/X86ISelLowering.h
===================================================================
--- lib/Target/X86/X86ISelLowering.h
+++ lib/Target/X86/X86ISelLowering.h
@@ -126,6 +126,9 @@
/// 1 is the number of bytes of stack to pop.
RET_FLAG,
+ /// Return from interrupt. Operand 0 is the number of bytes to pop.
+ IRET,
+
/// Repeat fill, corresponds to X86::REP_STOSx.
REP_STOS,
Index: lib/Target/X86/X86ISelLowering.cpp
===================================================================
--- lib/Target/X86/X86ISelLowering.cpp
+++ lib/Target/X86/X86ISelLowering.cpp
@@ -2057,6 +2057,9 @@
MachineFunction &MF = DAG.getMachineFunction();
X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
+ if (CallConv == CallingConv::X86_INTR && !Outs.empty())
+ report_fatal_error("X86 interrupts may not return any value");
+
SmallVector<CCValAssign, 16> RVLocs;
CCState CCInfo(CallConv, isVarArg, MF, RVLocs, *DAG.getContext());
CCInfo.AnalyzeReturn(Outs, RetCC_X86);
@@ -2170,7 +2173,10 @@
if (Flag.getNode())
RetOps.push_back(Flag);
- return DAG.getNode(X86ISD::RET_FLAG, dl, MVT::Other, RetOps);
+ X86ISD::NodeType opcode = X86ISD::RET_FLAG;
+ if (CallConv == CallingConv::X86_INTR)
+ opcode = X86ISD::IRET;
+ return DAG.getNode(opcode, dl, MVT::Other, RetOps);
}
bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
@@ -2412,6 +2418,14 @@
} else {
int FI = MFI->CreateFixedObject(ValVT.getSizeInBits()/8,
VA.getLocMemOffset(), isImmutable);
+ // Adjust SP offset of interrupt parameter, filling the slot
+ // normally taken by a return address.
+ if (CallConv == CallingConv::X86_INTR) {
+ const X86Subtarget& Subtarget =
+ static_cast<const X86Subtarget&>(DAG.getSubtarget());
+ MFI->setObjectOffset(FI, Subtarget.is64Bit() ? -8 : -4);
+ }
+
SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
SDValue Val = DAG.getLoad(
ValVT, dl, Chain, FIN,
@@ -2496,6 +2510,13 @@
assert(!(isVarArg && IsTailCallConvention(CallConv)) &&
"Var args not supported with calling convention fastcc, ghc or hipe");
+ if (CallConv == CallingConv::X86_INTR) {
+ bool isLegal = Ins.size() == 0
+ || (Ins.size() == 1 && Ins[0].VT == MVT::i32);
+ if (!isLegal)
+ report_fatal_error("X86 interrupts may take only one i32 argument");
+ }
+
// Assign locations to all of the incoming arguments.
SmallVector<CCValAssign, 16> ArgLocs;
CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
@@ -2782,6 +2803,9 @@
if (X86::isCalleePop(CallConv, Is64Bit, isVarArg,
MF.getTarget().Options.GuaranteedTailCallOpt)) {
FuncInfo->setBytesToPopOnReturn(StackSize); // Callee pops everything.
+ } else if (CallConv == CallingConv::X86_INTR && Ins.size() != 0) {
+ // Interrupts must pop the error code if present
+ FuncInfo->setBytesToPopOnReturn(Is64Bit ? 8 : 4);
} else {
FuncInfo->setBytesToPopOnReturn(0); // Callee pops nothing.
// If this is an sret function, the return should pop the hidden pointer.
@@ -2914,6 +2938,9 @@
X86MachineFunctionInfo *X86Info = MF.getInfo<X86MachineFunctionInfo>();
auto Attr = MF.getFunction()->getFnAttribute("disable-tail-calls");
+ if (CallConv == CallingConv::X86_INTR)
+ report_fatal_error("X86 interrupts may not be called directly");
+
if (Attr.getValueAsString() == "true")
isTailCall = false;
@@ -19221,6 +19248,7 @@
case X86ISD::CMOV: return "X86ISD::CMOV";
case X86ISD::BRCOND: return "X86ISD::BRCOND";
case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG";
+ case X86ISD::IRET: return "X86ISD::IRET";
case X86ISD::REP_STOS: return "X86ISD::REP_STOS";
case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS";
case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg";
Index: lib/Target/X86/X86InstrControl.td
===================================================================
--- lib/Target/X86/X86InstrControl.td
+++ lib/Target/X86/X86InstrControl.td
@@ -53,6 +53,19 @@
"{l}ret{|f}q\t$amt", [], IIC_RET>, Requires<[In64BitMode]>;
def LRETIW : Ii16<0xCA, RawFrm, (outs), (ins i16imm:$amt),
"{l}ret{w|f}\t$amt", [], IIC_RET>, OpSize16;
+
+ // The machine return from interrupt instruction, but sometimes we need to
+ // perform a post-epilogue stack adjustment. Codegen emits the pseudo form
+ // which expands to include an SP adjustment if necessary.
+ def IRET16 : I <0xcf, RawFrm, (outs), (ins), "iret{w}", [], IIC_IRET>,
+ OpSize16;
+ def IRET32 : I <0xcf, RawFrm, (outs), (ins), "iret{l|d}", [],
+ IIC_IRET>, OpSize32;
+ def IRET64 : RI <0xcf, RawFrm, (outs), (ins), "iretq", [],
+ IIC_IRET>, Requires<[In64BitMode]>;
+ let isCodeGenOnly = 1 in
+ def IRET : PseudoI<(outs), (ins i16imm:$adj), [(X86iret timm:$adj)]>;
+
}
// Unconditional branches.
Index: lib/Target/X86/X86InstrInfo.td
===================================================================
--- lib/Target/X86/X86InstrInfo.td
+++ lib/Target/X86/X86InstrInfo.td
@@ -156,6 +156,8 @@
def X86retflag : SDNode<"X86ISD::RET_FLAG", SDTX86Ret,
[SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
+def X86iret : SDNode<"X86ISD::IRET", SDTX86Ret,
+ [SDNPHasChain, SDNPOptInGlue]>;
def X86vastart_save_xmm_regs :
SDNode<"X86ISD::VASTART_SAVE_XMM_REGS",
Index: lib/Target/X86/X86InstrSystem.td
===================================================================
--- lib/Target/X86/X86InstrSystem.td
+++ lib/Target/X86/X86InstrSystem.td
@@ -60,12 +60,6 @@
IIC_SYS_ENTER_EXIT>, TB;
def SYSEXIT64 :RI<0x35, RawFrm, (outs), (ins), "sysexit{q}", [],
IIC_SYS_ENTER_EXIT>, TB, Requires<[In64BitMode]>;
-
-def IRET16 : I<0xcf, RawFrm, (outs), (ins), "iret{w}", [], IIC_IRET>, OpSize16;
-def IRET32 : I<0xcf, RawFrm, (outs), (ins), "iret{l|d}", [], IIC_IRET>,
- OpSize32;
-def IRET64 : RI<0xcf, RawFrm, (outs), (ins), "iretq", [], IIC_IRET>,
- Requires<[In64BitMode]>;
} // SchedRW
def : Pat<(debugtrap),
Index: lib/Target/X86/X86RegisterInfo.cpp
===================================================================
--- lib/Target/X86/X86RegisterInfo.cpp
+++ lib/Target/X86/X86RegisterInfo.cpp
@@ -216,6 +216,7 @@
const MCPhysReg *
X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
const X86Subtarget &Subtarget = MF->getSubtarget<X86Subtarget>();
+ bool HasSSE = Subtarget.hasSSE1();
bool HasAVX = Subtarget.hasAVX();
bool HasAVX512 = Subtarget.hasAVX512();
bool CallsEHReturn = MF->getMMI().callsEHReturn();
@@ -258,6 +259,18 @@
if (CallsEHReturn)
return CSR_64EHRet_SaveList;
return CSR_64_SaveList;
+ case CallingConv::X86_INTR:
+ if (Is64Bit) {
+ if (HasAVX)
+ return CSR_64_AllRegs_AVX_SaveList;
+ else
+ return CSR_64_AllRegs_SaveList;
+ } else {
+ if (HasSSE)
+ return CSR_32_AllRegs_SSE_SaveList;
+ else
+ return CSR_32_AllRegs_SaveList;
+ }
default:
break;
}
@@ -278,6 +291,7 @@
X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
CallingConv::ID CC) const {
const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
+ bool HasSSE = Subtarget.hasSSE1();
bool HasAVX = Subtarget.hasAVX();
bool HasAVX512 = Subtarget.hasAVX512();
@@ -312,12 +326,24 @@
if (Is64Bit)
return CSR_64_MostRegs_RegMask;
break;
- default:
- break;
case CallingConv::X86_64_Win64:
return CSR_Win64_RegMask;
case CallingConv::X86_64_SysV:
return CSR_64_RegMask;
+ case CallingConv::X86_INTR:
+ if (Is64Bit) {
+ if (HasAVX)
+ return CSR_64_AllRegs_AVX_RegMask;
+ else
+ return CSR_64_AllRegs_RegMask;
+ } else {
+ if (HasSSE)
+ return CSR_32_AllRegs_SSE_RegMask;
+ else
+ return CSR_32_AllRegs_RegMask;
+ }
+ default:
+ break;
}
// Unlike getCalleeSavedRegs(), we don't have MMI so we can't check
Index: test/CodeGen/X86/x86-intrcc.ll
===================================================================
--- /dev/null
+++ test/CodeGen/X86/x86-intrcc.ll
@@ -0,0 +1,61 @@
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s | FileCheck %s -check-prefix=X64
+; RUN: llc -mtriple=i686-unknown-unknown < %s | FileCheck %s -check-prefix=X32
+
+; No stack adjustment if declared with no parameters
+define x86_intrcc void @test_isr_noparams() {
+ ; CHECK-LABEL: test_isr_noparams:
+ ; X32: iretl
+ ; X64: iretq
+ ret void
+}
+
+; Spills rax, putting original rsp at +8. Stack is adjusted up another 8 bytes
+; before return, popping the error code.
+define x86_intrcc void @test_isr_oneparam(i32 %ecode) {
+ ; CHECK-LABEL: test_isr_oneparam
+ ; X64: pushq %rax
+ ; X64: movl 8(%rsp), %eax
+ ; X64: popq %rax
+ ; X64: addq $8, %rsp
+ ; X64: iretq
+ ; X32: pushl %eax
+ ; X32: movl 4(%ebp), %eax
+ ; X32: popl %eax
+ ; X32: addl $4, %esp
+ ; X32: iretl
+ call void asm sideeffect "movl $0, %eax", "m,~{eax}"(i32 %ecode)
+ ret void
+}
+
+; All clobbered registers must be saved
+define x86_intrcc void @test_isr_clobbers64(i32 %ecode) {
+ call void asm sideeffect "", "~{rax},~{rbx},~{rbp},~{r11},~{xmm0}"()
+ ; LABEL: test_isr_clobbers64
+ ; X64-SSE-NEXT: movaps %xmm0
+ ; X64-SSE-NEXT: pushq %rbx
+ ; X64-SSE-NEXT: pushq %rbp
+ ; X64-SSE-NEXT; pushq %r11
+ ; X64-SSE-NEXT: pushq %rax
+ ; X64-SSE-NEXT: popq %rax
+ ; X64-SSE-NEXT: popq %r11
+ ; X64-SSE-NEXT: popq %rbp
+ ; X64-SSE-NEXT: popq %rbx
+ ; X64-SSE-NEXT: movaps %xmm0
+ ; X64: addq $8, %rsp
+ ; X64-NEXT: iretq
+ ret void
+}
+
+define x86_intrcc void @test_isr_clobbers32(i32 %ecode) {
+ call void asm sideeffect "", "~{eax},~{ebx},~{ebp}"()
+ ; LABEL: test_isr_clobbers32
+ ; X32: pushl %ebp
+ ; X32: pushl %ebx
+ ; X32; pushl %eax
+ ; X32: popl %eax
+ ; X32: popl %ebx
+ ; X32: popl %ebp
+ ; X32: addl $4, %esp
+ ; X32-NEXT: iretl
+ ret void
+}