Index: include/llvm/CodeGen/SelectionDAG.h =================================================================== --- include/llvm/CodeGen/SelectionDAG.h +++ include/llvm/CodeGen/SelectionDAG.h @@ -755,6 +755,12 @@ /// the target's desired shift amount type. SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op); + /// \brief Expand the given \c ISD::VAARG node as the Legalize pass would. + SDValue expandVAArg(SDNode *Node); + + /// \brief Expand the given \c ISD::VACOPY node as the Legalize pass would. + SDValue expandVACopy(SDNode *Node); + /// UpdateNodeOperands - *Mutate* the specified node in-place to have the /// specified operands. If the resultant node already exists in the DAG, /// this does not modify the specified node, instead it returns the node that Index: lib/CodeGen/SelectionDAG/LegalizeDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -2933,57 +2933,13 @@ Results.push_back(Tmp1); break; } - case ISD::VAARG: { - const Value *V = cast(Node->getOperand(2))->getValue(); - EVT VT = Node->getValueType(0); - Tmp1 = Node->getOperand(0); - Tmp2 = Node->getOperand(1); - unsigned Align = Node->getConstantOperandVal(3); - - SDValue VAListLoad = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2, - MachinePointerInfo(V), - false, false, false, 0); - SDValue VAList = VAListLoad; - - if (Align > TLI.getMinStackArgumentAlignment()) { - assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2"); - - VAList = DAG.getNode(ISD::ADD, dl, VAList.getValueType(), VAList, - DAG.getConstant(Align - 1, - VAList.getValueType())); - - VAList = DAG.getNode(ISD::AND, dl, VAList.getValueType(), VAList, - DAG.getConstant(-(int64_t)Align, - VAList.getValueType())); - } - - // Increment the pointer, VAList, to the next vaarg - Tmp3 = DAG.getNode(ISD::ADD, dl, VAList.getValueType(), VAList, - DAG.getConstant(TLI.getDataLayout()-> - getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext())), - VAList.getValueType())); - // Store the incremented VAList to the legalized pointer - Tmp3 = DAG.getStore(VAListLoad.getValue(1), dl, Tmp3, Tmp2, - MachinePointerInfo(V), false, false, 0); - // Load the actual argument out of the pointer VAList - Results.push_back(DAG.getLoad(VT, dl, Tmp3, VAList, MachinePointerInfo(), - false, false, false, 0)); + case ISD::VAARG: + Results.push_back(DAG.expandVAArg(Node)); Results.push_back(Results[0].getValue(1)); break; - } - case ISD::VACOPY: { - // This defaults to loading a pointer from the input and storing it to the - // output, returning the chain. - const Value *VD = cast(Node->getOperand(3))->getValue(); - const Value *VS = cast(Node->getOperand(4))->getValue(); - Tmp1 = DAG.getLoad(TLI.getPointerTy(), dl, Node->getOperand(0), - Node->getOperand(2), MachinePointerInfo(VS), - false, false, false, 0); - Tmp1 = DAG.getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1), - MachinePointerInfo(VD), false, false, 0); - Results.push_back(Tmp1); + case ISD::VACOPY: + Results.push_back(DAG.expandVACopy(Node)); break; - } case ISD::EXTRACT_VECTOR_ELT: if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) // This must be an access of the only element. Return it. Index: lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -1515,6 +1515,56 @@ return getNode(Opcode, SDLoc(Op), ShTy, Op); } +SDValue SelectionDAG::expandVAArg(SDNode *Node) { + SDLoc dl(Node); + const TargetLowering &TLI = getTargetLoweringInfo(); + const Value *V = cast(Node->getOperand(2))->getValue(); + EVT VT = Node->getValueType(0); + SDValue Tmp1 = Node->getOperand(0); + SDValue Tmp2 = Node->getOperand(1); + unsigned Align = Node->getConstantOperandVal(3); + + SDValue VAListLoad = getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2, + MachinePointerInfo(V), false, false, false, 0); + SDValue VAList = VAListLoad; + + if (Align > TLI.getMinStackArgumentAlignment()) { + assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2"); + + VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, + getConstant(Align - 1, VAList.getValueType())); + + VAList = getNode(ISD::AND, dl, VAList.getValueType(), VAList, + getConstant(-(int64_t)Align, VAList.getValueType())); + } + + // Increment the pointer, VAList, to the next vaarg + Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, + getConstant(TLI.getDataLayout()-> + getTypeAllocSize(VT.getTypeForEVT(*getContext())), + VAList.getValueType())); + // Store the incremented VAList to the legalized pointer + Tmp1 = getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, + MachinePointerInfo(V), false, false, 0); + // Load the actual argument out of the pointer VAList + return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo(), + false, false, false, 0); +} + +SDValue SelectionDAG::expandVACopy(SDNode *Node) { + SDLoc dl(Node); + const TargetLowering &TLI = getTargetLoweringInfo(); + // This defaults to loading a pointer from the input and storing it to the + // output, returning the chain. + const Value *VD = cast(Node->getOperand(3))->getValue(); + const Value *VS = cast(Node->getOperand(4))->getValue(); + SDValue Tmp1 = getLoad(TLI.getPointerTy(), dl, Node->getOperand(0), + Node->getOperand(2), MachinePointerInfo(VS), + false, false, false, 0); + return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1), + MachinePointerInfo(VD), false, false, 0); +} + /// CreateStackTemporary - Create a stack temporary, suitable for holding the /// specified value type. SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -10779,7 +10779,8 @@ const Value *SV = cast(Op.getOperand(2))->getValue(); SDLoc DL(Op); - if (!Subtarget->is64Bit() || Subtarget->isTargetWin64()) { + if (!Subtarget->is64Bit() || Subtarget->isTargetWin64() || + MF.getFunction()->getCallingConv() == CallingConv::X86_64_Win64) { // vastart just stores the address of the VarArgsFrameIndex slot into the // memory location argument. SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), @@ -10840,6 +10841,12 @@ Subtarget->isTargetDarwin()) && "Unhandled target in LowerVAARG"); assert(Op.getNode()->getNumOperands() == 4); + + if (DAG.getMachineFunction().getFunction()->getCallingConv() == + CallingConv::X86_64_Win64) + // The Win64 ABI uses char* instead of a structure. + return DAG.expandVAArg(Op.getNode()); + SDValue Chain = Op.getOperand(0); SDValue SrcPtr = Op.getOperand(1); const Value *SV = cast(Op.getOperand(2))->getValue(); @@ -10903,6 +10910,9 @@ static SDValue LowerVACOPY(SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) { + if (DAG.getMachineFunction().getFunction()->getCallingConv() == + CallingConv::X86_64_Win64) + return DAG.expandVACopy(Op.getNode()); // X86-64 va_list is a struct { i32, i32, i8*, i8* }. assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!"); SDValue Chain = Op.getOperand(0); Index: lib/Transforms/Instrumentation/MemorySanitizer.cpp =================================================================== --- lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -1921,6 +1921,9 @@ } void visitVAStartInst(VAStartInst &I) { + if (I.getParent()->getParent()->getCallingConv() == + CallingConv::X86_64_Win64) + return; IRBuilder<> IRB(&I); VAStartInstrumentationList.push_back(&I); Value *VAListTag = I.getArgOperand(0); @@ -1933,6 +1936,9 @@ } void visitVACopyInst(VACopyInst &I) { + if (I.getParent()->getParent()->getCallingConv() == + CallingConv::X86_64_Win64) + return; IRBuilder<> IRB(&I); Value *VAListTag = I.getArgOperand(0); Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB); Index: test/CodeGen/X86/x86-64-ms_abi-vararg.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/x86-64-ms_abi-vararg.ll @@ -0,0 +1,103 @@ +; RUN: llc < %s -mcpu=generic -mtriple=x86_64-pc-linux-gnu | FileCheck %s + +; Verify that the var arg parameters which are passed in registers are stored +; in home stack slots allocated by the caller and that AP is correctly +; calculated. +define x86_64_win64cc void @average_va(i32 %count, ...) nounwind { +entry: +; CHECK: movq %r9, 32(%rsp) +; CHECK: movq %r8, 24(%rsp) +; CHECK: movq %rdx, 16(%rsp) +; CHECK: leaq 16(%rsp), %rax + + %ap = alloca i8*, align 8 ; [#uses=1] + %ap1 = bitcast i8** %ap to i8* ; [#uses=1] + call void @llvm.va_start(i8* %ap1) + ret void +} + +declare void @llvm.va_start(i8*) nounwind +declare void @llvm.va_copy(i8*, i8*) nounwind + +; CHECK-LABEL: f5: +; CHECK: leaq 48(%rsp), +define x86_64_win64cc i8* @f5(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, ...) nounwind { +entry: + %ap = alloca i8*, align 8 + %ap1 = bitcast i8** %ap to i8* + call void @llvm.va_start(i8* %ap1) + ret i8* %ap1 +} + +; CHECK-LABEL: f4: +; CHECK: leaq 40(%rsp), +define x86_64_win64cc i8* @f4(i64 %a0, i64 %a1, i64 %a2, i64 %a3, ...) nounwind { +entry: + %ap = alloca i8*, align 8 + %ap1 = bitcast i8** %ap to i8* + call void @llvm.va_start(i8* %ap1) + ret i8* %ap1 +} + +; CHECK-LABEL: f3: +; CHECK: leaq 32(%rsp), +define x86_64_win64cc i8* @f3(i64 %a0, i64 %a1, i64 %a2, ...) nounwind { +entry: + %ap = alloca i8*, align 8 + %ap1 = bitcast i8** %ap to i8* + call void @llvm.va_start(i8* %ap1) + ret i8* %ap1 +} + +; WinX86_64 uses char* for va_list. Verify that the correct amount of bytes +; are copied using va_copy. + +; CHECK-LABEL: copy1: +; CHECK: leaq 16(%rsp), [[REG_copy1:%[a-z]+]] +; CHECK: movq [[REG_copy1]], -8(%rsp) +; CHECK: movq [[REG_copy1]], -16(%rsp) +; CHECK: ret +define x86_64_win64cc void @copy1(i64 %a0, ...) nounwind { +entry: + %ap = alloca i8*, align 8 + %cp = alloca i8*, align 8 + %ap1 = bitcast i8** %ap to i8* + %cp1 = bitcast i8** %cp to i8* + call void @llvm.va_start(i8* %ap1) + call void @llvm.va_copy(i8* %cp1, i8* %ap1) + ret void +} + +; CHECK-LABEL: copy4: +; CHECK: leaq 40(%rsp), [[REG_copy4:%[a-z]+]] +; CHECK: movq [[REG_copy4]], -8(%rsp) +; CHECK: movq [[REG_copy4]], -16(%rsp) +; CHECK: ret +define x86_64_win64cc void @copy4(i64 %a0, i64 %a1, i64 %a2, i64 %a3, ...) nounwind { +entry: + %ap = alloca i8*, align 8 + %cp = alloca i8*, align 8 + %ap1 = bitcast i8** %ap to i8* + %cp1 = bitcast i8** %cp to i8* + call void @llvm.va_start(i8* %ap1) + call void @llvm.va_copy(i8* %cp1, i8* %ap1) + ret void +} + +; CHECK-LABEL: arg4: +; va_start: +; CHECK: leaq 40(%rsp), [[REG_arg4_1:%[a-z]+]] +; CHECK: movq [[REG_arg4_1]], -8(%rsp) +; va_arg: +; CHECK: leaq 44(%rsp), [[REG_arg4_2:%[a-z]+]] +; CHECK: movq [[REG_arg4_2]], -8(%rsp) +; CHECK: movl 40(%rsp), %eax +; CHECK: ret +define x86_64_win64cc i32 @arg4(i64 %a0, i64 %a1, i64 %a2, i64 %a3, ...) nounwind { +entry: + %ap = alloca i8*, align 8 + %ap1 = bitcast i8** %ap to i8* + call void @llvm.va_start(i8* %ap1) + %tmp = va_arg i8** %ap, i32 + ret i32 %tmp +}