diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h index 1eec08f51062..ef93042f6690 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -1,374 +1,380 @@ //===- llvm/CodeGen/GlobalISel/CallLowering.h - Call lowering ---*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// /// \file /// This file describes how to lower LLVM calls to machine code calls. /// //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_GLOBALISEL_CALLLOWERING_H #define LLVM_CODEGEN_GLOBALISEL_CALLLOWERING_H #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/TargetCallingConv.h" #include "llvm/IR/CallingConv.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MachineValueType.h" #include #include namespace llvm { class CallBase; class DataLayout; class Function; class MachineIRBuilder; class MachineOperand; struct MachinePointerInfo; class MachineRegisterInfo; class TargetLowering; class Type; class Value; class CallLowering { const TargetLowering *TLI; virtual void anchor(); public: struct ArgInfo { SmallVector Regs; // If the argument had to be split into multiple parts according to the // target calling convention, then this contains the original vregs // if the argument was an incoming arg. SmallVector OrigRegs; Type *Ty; SmallVector Flags; bool IsFixed; ArgInfo(ArrayRef Regs, Type *Ty, ArrayRef Flags = ArrayRef(), bool IsFixed = true) : Regs(Regs.begin(), Regs.end()), Ty(Ty), Flags(Flags.begin(), Flags.end()), IsFixed(IsFixed) { if (!Regs.empty() && Flags.empty()) this->Flags.push_back(ISD::ArgFlagsTy()); // FIXME: We should have just one way of saying "no register". assert(((Ty->isVoidTy() || Ty->isEmptyTy()) == (Regs.empty() || Regs[0] == 0)) && "only void types should have no register"); } ArgInfo() : Ty(nullptr), IsFixed(false) {} }; struct CallLoweringInfo { /// Calling convention to be used for the call. CallingConv::ID CallConv = CallingConv::C; /// Destination of the call. It should be either a register, globaladdress, /// or externalsymbol. MachineOperand Callee = MachineOperand::CreateImm(0); /// Descriptor for the return type of the function. ArgInfo OrigRet; /// List of descriptors of the arguments passed to the function. SmallVector OrigArgs; /// Valid if the call has a swifterror inout parameter, and contains the /// vreg that the swifterror should be copied into after the call. Register SwiftErrorVReg; MDNode *KnownCallees = nullptr; /// True if the call must be tail call optimized. bool IsMustTailCall = false; /// True if the call passes all target-independent checks for tail call /// optimization. bool IsTailCall = false; /// True if the call was lowered as a tail call. This is consumed by the /// legalizer. This allows the legalizer to lower libcalls as tail calls. bool LoweredTailCall = false; /// True if the call is to a vararg function. bool IsVarArg = false; }; /// Argument handling is mostly uniform between the four places that /// make these decisions: function formal arguments, call /// instruction args, call instruction returns and function /// returns. However, once a decision has been made on where an /// argument should go, exactly what happens can vary slightly. This /// class abstracts the differences. struct ValueHandler { ValueHandler(bool IsIncoming, MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, CCAssignFn *AssignFn) : MIRBuilder(MIRBuilder), MRI(MRI), AssignFn(AssignFn), IsIncomingArgumentHandler(IsIncoming) {} virtual ~ValueHandler() = default; /// Returns true if the handler is dealing with incoming arguments, /// i.e. those that move values from some physical location to vregs. bool isIncomingArgumentHandler() const { return IsIncomingArgumentHandler; } /// Materialize a VReg containing the address of the specified /// stack-based object. This is either based on a FrameIndex or /// direct SP manipulation, depending on the context. \p MPO /// should be initialized to an appropriate description of the /// address created. virtual Register getStackAddress(uint64_t Size, int64_t Offset, MachinePointerInfo &MPO) = 0; /// The specified value has been assigned to a physical register, /// handle the appropriate COPY (either to or from) and mark any /// relevant uses/defines as needed. virtual void assignValueToReg(Register ValVReg, Register PhysReg, CCValAssign &VA) = 0; /// The specified value has been assigned to a stack /// location. Load or store it there, with appropriate extension /// if necessary. virtual void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, MachinePointerInfo &MPO, CCValAssign &VA) = 0; /// An overload which takes an ArgInfo if additional information about /// the arg is needed. virtual void assignValueToAddress(const ArgInfo &Arg, Register Addr, uint64_t Size, MachinePointerInfo &MPO, CCValAssign &VA) { assert(Arg.Regs.size() == 1); assignValueToAddress(Arg.Regs[0], Addr, Size, MPO, VA); } /// Handle custom values, which may be passed into one or more of \p VAs. /// \return The number of \p VAs that have been assigned after the first /// one, and which should therefore be skipped from further /// processing. virtual unsigned assignCustomValue(const ArgInfo &Arg, ArrayRef VAs) { // This is not a pure virtual method because not all targets need to worry // about custom values. llvm_unreachable("Custom values not supported"); } /// Extend a register to the location type given in VA, capped at extending /// to at most MaxSize bits. If MaxSizeBits is 0 then no maximum is set. Register extendRegister(Register ValReg, CCValAssign &VA, unsigned MaxSizeBits = 0); virtual bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, const ArgInfo &Info, ISD::ArgFlagsTy Flags, CCState &State) { return AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State); } MachineIRBuilder &MIRBuilder; MachineRegisterInfo &MRI; CCAssignFn *AssignFn; private: bool IsIncomingArgumentHandler; virtual void anchor(); }; struct IncomingValueHandler : public ValueHandler { IncomingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, CCAssignFn *AssignFn) : ValueHandler(true, MIRBuilder, MRI, AssignFn) {} }; struct OutgoingValueHandler : public ValueHandler { OutgoingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, CCAssignFn *AssignFn) : ValueHandler(false, MIRBuilder, MRI, AssignFn) {} }; protected: /// Getter for generic TargetLowering class. const TargetLowering *getTLI() const { return TLI; } /// Getter for target specific TargetLowering class. template const XXXTargetLowering *getTLI() const { return static_cast(TLI); } /// \returns Flags corresponding to the attributes on the \p ArgIdx-th /// parameter of \p Call. ISD::ArgFlagsTy getAttributesForArgIdx(const CallBase &Call, unsigned ArgIdx) const; + /// Adds flags to \p Flags based off of the attributes in \p Attrs. + /// \p OpIdx is the index in \p Attrs to add flags from. + void addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags, + const AttributeList &Attrs, + unsigned OpIdx) const; + template void setArgFlags(ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL, const FuncInfoTy &FuncInfo) const; /// Generate instructions for packing \p SrcRegs into one big register /// corresponding to the aggregate type \p PackedTy. /// /// \param SrcRegs should contain one virtual register for each base type in /// \p PackedTy, as returned by computeValueLLTs. /// /// \return The packed register. Register packRegs(ArrayRef SrcRegs, Type *PackedTy, MachineIRBuilder &MIRBuilder) const; /// Generate instructions for unpacking \p SrcReg into the \p DstRegs /// corresponding to the aggregate type \p PackedTy. /// /// \param DstRegs should contain one virtual register for each base type in /// \p PackedTy, as returned by computeValueLLTs. void unpackRegs(ArrayRef DstRegs, Register SrcReg, Type *PackedTy, MachineIRBuilder &MIRBuilder) const; /// Invoke Handler::assignArg on each of the given \p Args and then use /// \p Handler to move them to the assigned locations. /// /// \return True if everything has succeeded, false otherwise. bool handleAssignments(MachineIRBuilder &MIRBuilder, SmallVectorImpl &Args, ValueHandler &Handler) const; bool handleAssignments(CCState &CCState, SmallVectorImpl &ArgLocs, MachineIRBuilder &MIRBuilder, SmallVectorImpl &Args, ValueHandler &Handler) const; /// Analyze passed or returned values from a call, supplied in \p ArgInfo, /// incorporating info about the passed values into \p CCState. /// /// Used to check if arguments are suitable for tail call lowering. bool analyzeArgInfo(CCState &CCState, SmallVectorImpl &Args, CCAssignFn &AssignFnFixed, CCAssignFn &AssignFnVarArg) const; /// \returns True if the calling convention for a callee and its caller pass /// results in the same way. Typically used for tail call eligibility checks. /// /// \p Info is the CallLoweringInfo for the call. /// \p MF is the MachineFunction for the caller. /// \p InArgs contains the results of the call. /// \p CalleeAssignFnFixed is the CCAssignFn to be used for the callee for /// fixed arguments. /// \p CalleeAssignFnVarArg is similar, but for varargs. /// \p CallerAssignFnFixed is the CCAssignFn to be used for the caller for /// fixed arguments. /// \p CallerAssignFnVarArg is similar, but for varargs. bool resultsCompatible(CallLoweringInfo &Info, MachineFunction &MF, SmallVectorImpl &InArgs, CCAssignFn &CalleeAssignFnFixed, CCAssignFn &CalleeAssignFnVarArg, CCAssignFn &CallerAssignFnFixed, CCAssignFn &CallerAssignFnVarArg) const; public: CallLowering(const TargetLowering *TLI) : TLI(TLI) {} virtual ~CallLowering() = default; /// \return true if the target is capable of handling swifterror values that /// have been promoted to a specified register. The extended versions of /// lowerReturn and lowerCall should be implemented. virtual bool supportSwiftError() const { return false; } /// This hook must be implemented to lower outgoing return values, described /// by \p Val, into the specified virtual registers \p VRegs. /// This hook is used by GlobalISel. /// /// \p SwiftErrorVReg is non-zero if the function has a swifterror parameter /// that needs to be implicitly returned. /// /// \return True if the lowering succeeds, false otherwise. virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, ArrayRef VRegs, Register SwiftErrorVReg) const { if (!supportSwiftError()) { assert(SwiftErrorVReg == 0 && "attempt to use unsupported swifterror"); return lowerReturn(MIRBuilder, Val, VRegs); } return false; } /// This hook behaves as the extended lowerReturn function, but for targets /// that do not support swifterror value promotion. virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, ArrayRef VRegs) const { return false; } virtual bool fallBackToDAGISel(const Function &F) const { return false; } /// This hook must be implemented to lower the incoming (formal) /// arguments, described by \p VRegs, for GlobalISel. Each argument /// must end up in the related virtual registers described by \p VRegs. /// In other words, the first argument should end up in \c VRegs[0], /// the second in \c VRegs[1], and so on. For each argument, there will be one /// register for each non-aggregate type, as returned by \c computeValueLLTs. /// \p MIRBuilder is set to the proper insertion for the argument /// lowering. /// /// \return True if the lowering succeeded, false otherwise. virtual bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, ArrayRef> VRegs) const { return false; } /// This hook must be implemented to lower the given call instruction, /// including argument and return value marshalling. /// /// /// \return true if the lowering succeeded, false otherwise. virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info) const { return false; } /// Lower the given call instruction, including argument and return value /// marshalling. /// /// \p CI is the call/invoke instruction. /// /// \p ResRegs are the registers where the call's return value should be /// stored (or 0 if there is no return value). There will be one register for /// each non-aggregate type, as returned by \c computeValueLLTs. /// /// \p ArgRegs is a list of lists of virtual registers containing each /// argument that needs to be passed (argument \c i should be placed in \c /// ArgRegs[i]). For each argument, there will be one register for each /// non-aggregate type, as returned by \c computeValueLLTs. /// /// \p SwiftErrorVReg is non-zero if the call has a swifterror inout /// parameter, and contains the vreg that the swifterror should be copied into /// after the call. /// /// \p GetCalleeReg is a callback to materialize a register for the callee if /// the target determines it cannot jump to the destination based purely on \p /// CI. This might be because \p CI is indirect, or because of the limited /// range of an immediate jump. /// /// \return true if the lowering succeeded, false otherwise. bool lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &Call, ArrayRef ResRegs, ArrayRef> ArgRegs, Register SwiftErrorVReg, std::function GetCalleeReg) const; }; } // end namespace llvm #endif // LLVM_CODEGEN_GLOBALISEL_CALLLOWERING_H diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp index cf1059c67b4a..49d101a81e93 100644 --- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp @@ -1,550 +1,548 @@ //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// /// \file /// This file implements some simple delegations needed for call lowering. /// //===----------------------------------------------------------------------===// #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/GlobalISel/CallLowering.h" #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/Target/TargetMachine.h" #define DEBUG_TYPE "call-lowering" using namespace llvm; void CallLowering::anchor() {} -ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call, - unsigned ArgIdx) const { - ISD::ArgFlagsTy Flags; - if (Call.paramHasAttr(ArgIdx, Attribute::SExt)) +/// Helper function which updates \p Flags when \p AttrFn returns true. +static void +addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags, + const std::function &AttrFn) { + if (AttrFn(Attribute::SExt)) Flags.setSExt(); - if (Call.paramHasAttr(ArgIdx, Attribute::ZExt)) + if (AttrFn(Attribute::ZExt)) Flags.setZExt(); - if (Call.paramHasAttr(ArgIdx, Attribute::InReg)) + if (AttrFn(Attribute::InReg)) Flags.setInReg(); - if (Call.paramHasAttr(ArgIdx, Attribute::StructRet)) + if (AttrFn(Attribute::StructRet)) Flags.setSRet(); - if (Call.paramHasAttr(ArgIdx, Attribute::Nest)) + if (AttrFn(Attribute::Nest)) Flags.setNest(); - if (Call.paramHasAttr(ArgIdx, Attribute::ByVal)) + if (AttrFn(Attribute::ByVal)) Flags.setByVal(); - if (Call.paramHasAttr(ArgIdx, Attribute::Preallocated)) + if (AttrFn(Attribute::Preallocated)) Flags.setPreallocated(); - if (Call.paramHasAttr(ArgIdx, Attribute::InAlloca)) + if (AttrFn(Attribute::InAlloca)) Flags.setInAlloca(); - if (Call.paramHasAttr(ArgIdx, Attribute::Returned)) + if (AttrFn(Attribute::Returned)) Flags.setReturned(); - if (Call.paramHasAttr(ArgIdx, Attribute::SwiftSelf)) + if (AttrFn(Attribute::SwiftSelf)) Flags.setSwiftSelf(); - if (Call.paramHasAttr(ArgIdx, Attribute::SwiftError)) + if (AttrFn(Attribute::SwiftError)) Flags.setSwiftError(); +} + +ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call, + unsigned ArgIdx) const { + ISD::ArgFlagsTy Flags; + addFlagsUsingAttrFn(Flags, [&Call, &ArgIdx](Attribute::AttrKind Attr) { + return Call.paramHasAttr(ArgIdx, Attr); + }); return Flags; } +void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags, + const AttributeList &Attrs, + unsigned OpIdx) const { + addFlagsUsingAttrFn(Flags, [&Attrs, &OpIdx](Attribute::AttrKind Attr) { + return Attrs.hasAttribute(OpIdx, Attr); + }); +} + bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB, ArrayRef ResRegs, ArrayRef> ArgRegs, Register SwiftErrorVReg, std::function GetCalleeReg) const { CallLoweringInfo Info; const DataLayout &DL = MIRBuilder.getDataLayout(); MachineFunction &MF = MIRBuilder.getMF(); bool CanBeTailCalled = CB.isTailCall() && isInTailCallPosition(CB, MF.getTarget()) && (MF.getFunction() .getFnAttribute("disable-tail-calls") .getValueAsString() != "true"); // First step is to marshall all the function's parameters into the correct // physregs and memory locations. Gather the sequence of argument types that // we'll pass to the assigner function. unsigned i = 0; unsigned NumFixedArgs = CB.getFunctionType()->getNumParams(); for (auto &Arg : CB.args()) { ArgInfo OrigArg{ArgRegs[i], Arg->getType(), getAttributesForArgIdx(CB, i), i < NumFixedArgs}; setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB); // If we have an explicit sret argument that is an Instruction, (i.e., it // might point to function-local memory), we can't meaningfully tail-call. if (OrigArg.Flags[0].isSRet() && isa(&Arg)) CanBeTailCalled = false; Info.OrigArgs.push_back(OrigArg); ++i; } // Try looking through a bitcast from one function type to another. // Commonly happens with calls to objc_msgSend(). const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts(); if (const Function *F = dyn_cast(CalleeV)) Info.Callee = MachineOperand::CreateGA(F, 0); else Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false); Info.OrigRet = ArgInfo{ResRegs, CB.getType(), ISD::ArgFlagsTy{}}; if (!Info.OrigRet.Ty->isVoidTy()) setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CB); Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees); Info.CallConv = CB.getCallingConv(); Info.SwiftErrorVReg = SwiftErrorVReg; Info.IsMustTailCall = CB.isMustTailCall(); Info.IsTailCall = CanBeTailCalled; Info.IsVarArg = CB.getFunctionType()->isVarArg(); return lowerCall(MIRBuilder, Info); } template void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL, const FuncInfoTy &FuncInfo) const { auto &Flags = Arg.Flags[0]; const AttributeList &Attrs = FuncInfo.getAttributes(); - if (Attrs.hasAttribute(OpIdx, Attribute::ZExt)) - Flags.setZExt(); - if (Attrs.hasAttribute(OpIdx, Attribute::SExt)) - Flags.setSExt(); - if (Attrs.hasAttribute(OpIdx, Attribute::InReg)) - Flags.setInReg(); - if (Attrs.hasAttribute(OpIdx, Attribute::StructRet)) - Flags.setSRet(); - if (Attrs.hasAttribute(OpIdx, Attribute::SwiftSelf)) - Flags.setSwiftSelf(); - if (Attrs.hasAttribute(OpIdx, Attribute::SwiftError)) - Flags.setSwiftError(); - if (Attrs.hasAttribute(OpIdx, Attribute::ByVal)) - Flags.setByVal(); - if (Attrs.hasAttribute(OpIdx, Attribute::Preallocated)) - Flags.setPreallocated(); - if (Attrs.hasAttribute(OpIdx, Attribute::InAlloca)) - Flags.setInAlloca(); + addArgFlagsFromAttributes(Flags, Attrs, OpIdx); if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) { Type *ElementTy = cast(Arg.Ty)->getElementType(); auto Ty = Attrs.getAttribute(OpIdx, Attribute::ByVal).getValueAsType(); Flags.setByValSize(DL.getTypeAllocSize(Ty ? Ty : ElementTy)); // For ByVal, alignment should be passed from FE. BE will guess if // this info is not there but there are cases it cannot get right. Align FrameAlign; if (auto ParamAlign = FuncInfo.getParamAlign(OpIdx - 2)) FrameAlign = *ParamAlign; else FrameAlign = Align(getTLI()->getByValTypeAlignment(ElementTy, DL)); Flags.setByValAlign(FrameAlign); } - if (Attrs.hasAttribute(OpIdx, Attribute::Nest)) - Flags.setNest(); Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty)); } template void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL, const Function &FuncInfo) const; template void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL, const CallBase &FuncInfo) const; Register CallLowering::packRegs(ArrayRef SrcRegs, Type *PackedTy, MachineIRBuilder &MIRBuilder) const { assert(SrcRegs.size() > 1 && "Nothing to pack"); const DataLayout &DL = MIRBuilder.getMF().getDataLayout(); MachineRegisterInfo *MRI = MIRBuilder.getMRI(); LLT PackedLLT = getLLTForType(*PackedTy, DL); SmallVector LLTs; SmallVector Offsets; computeValueLLTs(DL, *PackedTy, LLTs, &Offsets); assert(LLTs.size() == SrcRegs.size() && "Regs / types mismatch"); Register Dst = MRI->createGenericVirtualRegister(PackedLLT); MIRBuilder.buildUndef(Dst); for (unsigned i = 0; i < SrcRegs.size(); ++i) { Register NewDst = MRI->createGenericVirtualRegister(PackedLLT); MIRBuilder.buildInsert(NewDst, Dst, SrcRegs[i], Offsets[i]); Dst = NewDst; } return Dst; } void CallLowering::unpackRegs(ArrayRef DstRegs, Register SrcReg, Type *PackedTy, MachineIRBuilder &MIRBuilder) const { assert(DstRegs.size() > 1 && "Nothing to unpack"); const DataLayout &DL = MIRBuilder.getDataLayout(); SmallVector LLTs; SmallVector Offsets; computeValueLLTs(DL, *PackedTy, LLTs, &Offsets); assert(LLTs.size() == DstRegs.size() && "Regs / types mismatch"); for (unsigned i = 0; i < DstRegs.size(); ++i) MIRBuilder.buildExtract(DstRegs[i], SrcReg, Offsets[i]); } bool CallLowering::handleAssignments(MachineIRBuilder &MIRBuilder, SmallVectorImpl &Args, ValueHandler &Handler) const { MachineFunction &MF = MIRBuilder.getMF(); const Function &F = MF.getFunction(); SmallVector ArgLocs; CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext()); return handleAssignments(CCInfo, ArgLocs, MIRBuilder, Args, Handler); } bool CallLowering::handleAssignments(CCState &CCInfo, SmallVectorImpl &ArgLocs, MachineIRBuilder &MIRBuilder, SmallVectorImpl &Args, ValueHandler &Handler) const { MachineFunction &MF = MIRBuilder.getMF(); const Function &F = MF.getFunction(); const DataLayout &DL = F.getParent()->getDataLayout(); unsigned NumArgs = Args.size(); for (unsigned i = 0; i != NumArgs; ++i) { EVT CurVT = EVT::getEVT(Args[i].Ty); if (CurVT.isSimple() && !Handler.assignArg(i, CurVT.getSimpleVT(), CurVT.getSimpleVT(), CCValAssign::Full, Args[i], Args[i].Flags[0], CCInfo)) continue; MVT NewVT = TLI->getRegisterTypeForCallingConv( F.getContext(), F.getCallingConv(), EVT(CurVT)); // If we need to split the type over multiple regs, check it's a scenario // we currently support. unsigned NumParts = TLI->getNumRegistersForCallingConv( F.getContext(), F.getCallingConv(), CurVT); if (NumParts > 1) { // For now only handle exact splits. if (NewVT.getSizeInBits() * NumParts != CurVT.getSizeInBits()) return false; } // For incoming arguments (physregs to vregs), we could have values in // physregs (or memlocs) which we want to extract and copy to vregs. // During this, we might have to deal with the LLT being split across // multiple regs, so we have to record this information for later. // // If we have outgoing args, then we have the opposite case. We have a // vreg with an LLT which we want to assign to a physical location, and // we might have to record that the value has to be split later. if (Handler.isIncomingArgumentHandler()) { if (NumParts == 1) { // Try to use the register type if we couldn't assign the VT. if (Handler.assignArg(i, NewVT, NewVT, CCValAssign::Full, Args[i], Args[i].Flags[0], CCInfo)) return false; } else { // We're handling an incoming arg which is split over multiple regs. // E.g. passing an s128 on AArch64. ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0]; Args[i].OrigRegs.push_back(Args[i].Regs[0]); Args[i].Regs.clear(); Args[i].Flags.clear(); LLT NewLLT = getLLTForMVT(NewVT); // For each split register, create and assign a vreg that will store // the incoming component of the larger value. These will later be // merged to form the final vreg. for (unsigned Part = 0; Part < NumParts; ++Part) { Register Reg = MIRBuilder.getMRI()->createGenericVirtualRegister(NewLLT); ISD::ArgFlagsTy Flags = OrigFlags; if (Part == 0) { Flags.setSplit(); } else { Flags.setOrigAlign(Align(1)); if (Part == NumParts - 1) Flags.setSplitEnd(); } Args[i].Regs.push_back(Reg); Args[i].Flags.push_back(Flags); if (Handler.assignArg(i + Part, NewVT, NewVT, CCValAssign::Full, Args[i], Args[i].Flags[Part], CCInfo)) { // Still couldn't assign this smaller part type for some reason. return false; } } } } else { // Handling an outgoing arg that might need to be split. if (NumParts < 2) return false; // Don't know how to deal with this type combination. // This type is passed via multiple registers in the calling convention. // We need to extract the individual parts. Register LargeReg = Args[i].Regs[0]; LLT SmallTy = LLT::scalar(NewVT.getSizeInBits()); auto Unmerge = MIRBuilder.buildUnmerge(SmallTy, LargeReg); assert(Unmerge->getNumOperands() == NumParts + 1); ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0]; // We're going to replace the regs and flags with the split ones. Args[i].Regs.clear(); Args[i].Flags.clear(); for (unsigned PartIdx = 0; PartIdx < NumParts; ++PartIdx) { ISD::ArgFlagsTy Flags = OrigFlags; if (PartIdx == 0) { Flags.setSplit(); } else { Flags.setOrigAlign(Align(1)); if (PartIdx == NumParts - 1) Flags.setSplitEnd(); } Args[i].Regs.push_back(Unmerge.getReg(PartIdx)); Args[i].Flags.push_back(Flags); if (Handler.assignArg(i + PartIdx, NewVT, NewVT, CCValAssign::Full, Args[i], Args[i].Flags[PartIdx], CCInfo)) return false; } } } for (unsigned i = 0, e = Args.size(), j = 0; i != e; ++i, ++j) { assert(j < ArgLocs.size() && "Skipped too many arg locs"); CCValAssign &VA = ArgLocs[j]; assert(VA.getValNo() == i && "Location doesn't correspond to current arg"); if (VA.needsCustom()) { unsigned NumArgRegs = Handler.assignCustomValue(Args[i], makeArrayRef(ArgLocs).slice(j)); if (!NumArgRegs) return false; j += NumArgRegs; continue; } // FIXME: Pack registers if we have more than one. Register ArgReg = Args[i].Regs[0]; EVT OrigVT = EVT::getEVT(Args[i].Ty); EVT VAVT = VA.getValVT(); const LLT OrigTy = getLLTForType(*Args[i].Ty, DL); // Expected to be multiple regs for a single incoming arg. // There should be Regs.size() ArgLocs per argument. unsigned NumArgRegs = Args[i].Regs.size(); assert((j + (NumArgRegs - 1)) < ArgLocs.size() && "Too many regs for number of args"); for (unsigned Part = 0; Part < NumArgRegs; ++Part) { // There should be Regs.size() ArgLocs per argument. VA = ArgLocs[j + Part]; if (VA.isMemLoc()) { // Don't currently support loading/storing a type that needs to be split // to the stack. Should be easy, just not implemented yet. if (NumArgRegs > 1) { LLVM_DEBUG( dbgs() << "Load/store a split arg to/from the stack not implemented yet\n"); return false; } // FIXME: Use correct address space for pointer size EVT LocVT = VA.getValVT(); unsigned MemSize = LocVT == MVT::iPTR ? DL.getPointerSize() : LocVT.getStoreSize(); unsigned Offset = VA.getLocMemOffset(); MachinePointerInfo MPO; Register StackAddr = Handler.getStackAddress(MemSize, Offset, MPO); Handler.assignValueToAddress(Args[i], StackAddr, MemSize, MPO, VA); continue; } assert(VA.isRegLoc() && "custom loc should have been handled already"); if (OrigVT.getSizeInBits() >= VAVT.getSizeInBits() || !Handler.isIncomingArgumentHandler()) { // This is an argument that might have been split. There should be // Regs.size() ArgLocs per argument. // Insert the argument copies. If VAVT < OrigVT, we'll insert the merge // to the original register after handling all of the parts. Handler.assignValueToReg(Args[i].Regs[Part], VA.getLocReg(), VA); continue; } // This ArgLoc covers multiple pieces, so we need to split it. const LLT VATy(VAVT.getSimpleVT()); Register NewReg = MIRBuilder.getMRI()->createGenericVirtualRegister(VATy); Handler.assignValueToReg(NewReg, VA.getLocReg(), VA); // If it's a vector type, we either need to truncate the elements // or do an unmerge to get the lower block of elements. if (VATy.isVector() && VATy.getNumElements() > OrigVT.getVectorNumElements()) { // Just handle the case where the VA type is 2 * original type. if (VATy.getNumElements() != OrigVT.getVectorNumElements() * 2) { LLVM_DEBUG(dbgs() << "Incoming promoted vector arg has too many elts"); return false; } auto Unmerge = MIRBuilder.buildUnmerge({OrigTy, OrigTy}, {NewReg}); MIRBuilder.buildCopy(ArgReg, Unmerge.getReg(0)); } else { MIRBuilder.buildTrunc(ArgReg, {NewReg}).getReg(0); } } // Now that all pieces have been handled, re-pack any arguments into any // wider, original registers. if (Handler.isIncomingArgumentHandler()) { if (VAVT.getSizeInBits() < OrigVT.getSizeInBits()) { assert(NumArgRegs >= 2); // Merge the split registers into the expected larger result vreg // of the original call. MIRBuilder.buildMerge(Args[i].OrigRegs[0], Args[i].Regs); } } j += NumArgRegs - 1; } return true; } bool CallLowering::analyzeArgInfo(CCState &CCState, SmallVectorImpl &Args, CCAssignFn &AssignFnFixed, CCAssignFn &AssignFnVarArg) const { for (unsigned i = 0, e = Args.size(); i < e; ++i) { MVT VT = MVT::getVT(Args[i].Ty); CCAssignFn &Fn = Args[i].IsFixed ? AssignFnFixed : AssignFnVarArg; if (Fn(i, VT, VT, CCValAssign::Full, Args[i].Flags[0], CCState)) { // Bail out on anything we can't handle. LLVM_DEBUG(dbgs() << "Cannot analyze " << EVT(VT).getEVTString() << " (arg number = " << i << "\n"); return false; } } return true; } bool CallLowering::resultsCompatible(CallLoweringInfo &Info, MachineFunction &MF, SmallVectorImpl &InArgs, CCAssignFn &CalleeAssignFnFixed, CCAssignFn &CalleeAssignFnVarArg, CCAssignFn &CallerAssignFnFixed, CCAssignFn &CallerAssignFnVarArg) const { const Function &F = MF.getFunction(); CallingConv::ID CalleeCC = Info.CallConv; CallingConv::ID CallerCC = F.getCallingConv(); if (CallerCC == CalleeCC) return true; SmallVector ArgLocs1; CCState CCInfo1(CalleeCC, false, MF, ArgLocs1, F.getContext()); if (!analyzeArgInfo(CCInfo1, InArgs, CalleeAssignFnFixed, CalleeAssignFnVarArg)) return false; SmallVector ArgLocs2; CCState CCInfo2(CallerCC, false, MF, ArgLocs2, F.getContext()); if (!analyzeArgInfo(CCInfo2, InArgs, CallerAssignFnFixed, CalleeAssignFnVarArg)) return false; // We need the argument locations to match up exactly. If there's more in // one than the other, then we are done. if (ArgLocs1.size() != ArgLocs2.size()) return false; // Make sure that each location is passed in exactly the same way. for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) { const CCValAssign &Loc1 = ArgLocs1[i]; const CCValAssign &Loc2 = ArgLocs2[i]; // We need both of them to be the same. So if one is a register and one // isn't, we're done. if (Loc1.isRegLoc() != Loc2.isRegLoc()) return false; if (Loc1.isRegLoc()) { // If they don't have the same register location, we're done. if (Loc1.getLocReg() != Loc2.getLocReg()) return false; // They matched, so we can move to the next ArgLoc. continue; } // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match. if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset()) return false; } return true; } Register CallLowering::ValueHandler::extendRegister(Register ValReg, CCValAssign &VA, unsigned MaxSizeBits) { LLT LocTy{VA.getLocVT()}; LLT ValTy = MRI.getType(ValReg); if (LocTy.getSizeInBits() == ValTy.getSizeInBits()) return ValReg; if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) { if (MaxSizeBits <= ValTy.getSizeInBits()) return ValReg; LocTy = LLT::scalar(MaxSizeBits); } switch (VA.getLocInfo()) { default: break; case CCValAssign::Full: case CCValAssign::BCvt: // FIXME: bitconverting between vector types may or may not be a // nop in big-endian situations. return ValReg; case CCValAssign::AExt: { auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg); return MIB.getReg(0); } case CCValAssign::SExt: { Register NewReg = MRI.createGenericVirtualRegister(LocTy); MIRBuilder.buildSExt(NewReg, ValReg); return NewReg; } case CCValAssign::ZExt: { Register NewReg = MRI.createGenericVirtualRegister(LocTy); MIRBuilder.buildZExt(NewReg, ValReg); return NewReg; } } llvm_unreachable("unable to extend register"); } void CallLowering::ValueHandler::anchor() {}