Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -1055,20 +1055,19 @@ "Attribute 'builtin' can only be applied to a callsite.", &F); // Check that this function meets the restrictions on this calling convention. + // Sometimes varargs is used for perfectly forwarding thunks, so some of these + // restrictions can be lifted. switch (F.getCallingConv()) { default: - break; case CallingConv::C: break; case CallingConv::Fast: case CallingConv::Cold: - case CallingConv::X86_FastCall: - case CallingConv::X86_ThisCall: case CallingConv::Intel_OCL_BI: case CallingConv::PTX_Kernel: case CallingConv::PTX_Device: - Assert1(!F.isVarArg(), - "Varargs functions must have C calling conventions!", &F); + Assert1(!F.isVarArg(), "Calling convention does not support varargs or " + "perfect forwarding!", &F); break; } @@ -1125,6 +1124,19 @@ "Function is marked as dllimport, but not external.", &F); } +static bool isCalleeCleanup(CallingConv::ID CC) { + switch (CC) { + case CallingConv::X86_FastCall: + case CallingConv::X86_StdCall: + case CallingConv::X86_ThisCall: + return true; + default: + case CallingConv::C: + return false; + } + llvm_unreachable("invalid enum"); +} + // verifyBasicBlock - Verify that a basic block is well formed... // void Verifier::visitBasicBlock(BasicBlock &BB) { @@ -1194,7 +1206,8 @@ } void Verifier::visitReturnInst(ReturnInst &RI) { - Function *F = RI.getParent()->getParent(); + BasicBlock *BB = RI.getParent(); + Function *F = BB->getParent(); unsigned N = RI.getNumOperands(); if (F->getReturnType()->isVoidTy()) Assert2(N == 0, @@ -1205,6 +1218,15 @@ "Function return type does not match operand " "type of return inst!", &RI, F->getReturnType()); + // Verify that varargs callee cleanup functions never do a normal return. All + // returns must be preceded by a musttail call, or be in trivially unreachable + // blocks. + if (F->isVarArg() && isCalleeCleanup(F->getCallingConv())) { + Assert1(BB->getTerminatingMustTailCall() != nullptr || + !DT.isReachableFromEntry(BB), + "Found impossible return in varargs callee cleanup function!", &RI); + } + // Check to make sure that the return value has necessary properties for // terminators... visitTerminatorInst(RI); Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -3544,23 +3544,18 @@ /// own arguments. Callee pop is necessary to support tail calls. bool X86::isCalleePop(CallingConv::ID CallingConv, bool is64Bit, bool IsVarArg, bool TailCallOpt) { - if (IsVarArg) - return false; - switch (CallingConv) { default: return false; case CallingConv::X86_StdCall: - return !is64Bit; case CallingConv::X86_FastCall: - return !is64Bit; case CallingConv::X86_ThisCall: return !is64Bit; case CallingConv::Fast: - return TailCallOpt; case CallingConv::GHC: - return TailCallOpt; case CallingConv::HiPE: + if (IsVarArg) + return false; return TailCallOpt; } } Index: test/CodeGen/X86/vararg-callee-cleanup.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/vararg-callee-cleanup.ll @@ -0,0 +1,54 @@ +; RUN: llc -mtriple=i686-pc-windows < %s | FileCheck %s + +target datalayout = "e-m:w-p:32:32-i64:64-f80:32-n8:16:32-S32" + +declare x86_thiscallcc void @thiscall_thunk(i8* %this, ...) +define i32 @call_varargs_thiscall_thunk(i8* %a, i32 %b, i32 %c, i32 %d) { + call x86_thiscallcc void (i8*, ...)* @thiscall_thunk(i8* %a, i32 1, i32 2) + call x86_thiscallcc void (i8*, ...)* @thiscall_thunk(i8* %a, i32 1, i32 2) + %t1 = add i32 %b, %c + %r = add i32 %t1, %d + ret i32 %r +} + +; CHECK: _call_varargs_thiscall_thunk: +; CHECK: calll _thiscall_thunk +; CHECK-NEXT: subl $8, %esp + +; We don't mangle the argument size into variadic callee cleanup functions. + +declare x86_stdcallcc void @stdcall_thunk(i8* %this, ...) +define i32 @call_varargs_stdcall_thunk(i8* %a, i32 %b, i32 %c, i32 %d) { + call x86_stdcallcc void (i8*, ...)* @stdcall_thunk(i8* %a, i32 1, i32 2) + call x86_stdcallcc void (i8*, ...)* @stdcall_thunk(i8* %a, i32 1, i32 2) + %t1 = add i32 %b, %c + %r = add i32 %t1, %d + ret i32 %r +} + +; CHECK: _call_varargs_stdcall_thunk: +; CHECK: calll _stdcall_thunk{{$}} +; CHECK-NEXT: subl $12, %esp + +declare x86_fastcallcc void @fastcall_thunk(i8* %this, ...) +define i32 @call_varargs_fastcall_thunk(i8* %a, i32 %b, i32 %c, i32 %d) { + call x86_fastcallcc void (i8*, ...)* @fastcall_thunk(i8* inreg %a, i32 inreg 1, i32 2) + call x86_fastcallcc void (i8*, ...)* @fastcall_thunk(i8* inreg %a, i32 inreg 1, i32 2) + %t1 = add i32 %b, %c + %r = add i32 %t1, %d + ret i32 %r +} + +; CHECK: _call_varargs_fastcall_thunk: +; CHECK: calll @fastcall_thunk{{$}} +; CHECK-NEXT: subl $4, %esp + +; If you actually return from such a thunk, it will only pop the non-variadic +; portion of the arguments, which is different from what the callee passes. + +define x86_stdcallcc void @varargs_stdcall_return(i32, i32, ...) { + ret void +} + +; CHECK: _varargs_stdcall_return: +; CHECK: retl $8 Index: test/Verifier/musttail-valid.ll =================================================================== --- test/Verifier/musttail-valid.ll +++ test/Verifier/musttail-valid.ll @@ -14,3 +14,28 @@ %w = bitcast i8* %v to i32* ret i32* %w } + +; These should verify. Only normal, reachable returns should fail verification. + +declare x86_thiscallcc void @varargs_thiscall(i8*, ...) +define x86_thiscallcc void @varargs_thiscall_thunk(i8* %this, ...) { + musttail call x86_thiscallcc void (i8*, ...)* @varargs_thiscall(i8* %this, ...) + ret void +} + +declare x86_fastcallcc void @varargs_fastcall(i8*, ...) +define x86_fastcallcc void @varargs_fastcall_thunk(i8* %this, ...) { + musttail call x86_fastcallcc void (i8*, ...)* @varargs_fastcall(i8* %this, ...) + ret void +} + +define x86_thiscallcc void @varargs_thiscall_unreachable(i8* %this, ...) { + unreachable +} + +define x86_thiscallcc void @varargs_thiscall_ret_unreachable(i8* %this, ...) { + musttail call x86_thiscallcc void (i8*, ...)* @varargs_thiscall(i8* %this, ...) + ret void +bb1: + ret void +} Index: test/Verifier/varargs-callee-cleanup.ll =================================================================== --- /dev/null +++ test/Verifier/varargs-callee-cleanup.ll @@ -0,0 +1,16 @@ +; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s + +define x86_stdcallcc void @stdcall_varargs(i8*, ...) { + ret void +} +; CHECK: Found impossible return in varargs callee cleanup function! + +define x86_thiscallcc void @thiscall_varargs(i8*, ...) { + ret void +} +; CHECK: Found impossible return in varargs callee cleanup function! + +define x86_fastcallcc void @fastcall_varargs(i8*, ...) { + ret void +} +; CHECK: Found impossible return in varargs callee cleanup function!