diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h --- a/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/llvm/include/llvm/CodeGen/MachineFunction.h @@ -28,6 +28,7 @@ #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/IR/Instructions.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/ArrayRecycler.h" #include "llvm/Support/AtomicOrdering.h" @@ -414,16 +415,49 @@ assert(Arg < (1 << 16) && "Arg out of range"); } }; - /// Vector of call argument and its forwarding register. - using CallSiteInfo = SmallVector; - using CallSiteInfoImpl = SmallVectorImpl; + + struct CallSiteInfo { + /// Vector of call argument and its forwarding register. + SmallVector ArgRegPairs; + /// Callee type id. + ConstantInt *TypeId = nullptr; + + /// Extracts a generalized numeric type identifier of a CallBase's type from + /// type operand bundle. Returned object can be used to set TypeId. Returns + /// null if such operand bundle cannot be found. + static ConstantInt *extractNumericCGTypeId(const CallBase &CB) { + // Call graph section needs numeric type id only for indirect calls. + if (!CB.isIndirectCall()) + return nullptr; + + auto Opt = CB.getOperandBundle(LLVMContext::OB_type); + if (!Opt.hasValue()) + return nullptr; + + // Get generalized type id string + auto OB = Opt.getValue(); + assert(OB.Inputs.size() == 1 && "invalid input size"); + auto *OBVal = OB.Inputs.front().get(); + assert(isa(OBVal) && "invalid value type"); + auto *TypeIdMD = cast(OBVal)->getMetadata(); + assert(isa(TypeIdMD) && "invalid metadata type"); + auto *TypeIdStr = cast(TypeIdMD); + assert(TypeIdStr->getString().endswith(".generalized") && + "invalid type identifier"); + + // Compute numeric type id from generalized type id string + uint64_t TypeIdVal = llvm::MD5Hash(TypeIdStr->getString()); + IntegerType *Int64Ty = Type::getInt64Ty(CB.getContext()); + return ConstantInt::get(Int64Ty, TypeIdVal, false); + } + }; private: Delegate *TheDelegate = nullptr; GISelChangeObserver *Observer = nullptr; using CallSiteInfoMap = DenseMap; - /// Map a call instruction to call site arguments forwarding info. + /// Map a call instruction to call site arguments forwarding and type id. CallSiteInfoMap CallSitesInfo; /// A helper function that returns call site info for a give call @@ -1172,9 +1206,8 @@ return VariableDbgInfos; } - /// Start tracking the arguments passed to the call \p CallI. - void addCallArgsForwardingRegs(const MachineInstr *CallI, - CallSiteInfoImpl &&CallInfo) { + /// Start tracking the arguments passed to the call \p CallI and call type. + void addCallSiteInfo(const MachineInstr *CallI, CallSiteInfo &&CallInfo) { assert(CallI->isCandidateForCallSiteEntry()); bool Inserted = CallSitesInfo.try_emplace(CallI, std::move(CallInfo)).second; diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -266,7 +266,6 @@ SDDbgInfo *DbgInfo; using CallSiteInfo = MachineFunction::CallSiteInfo; - using CallSiteInfoImpl = MachineFunction::CallSiteInfoImpl; struct CallSiteDbgInfo { CallSiteInfo CSInfo; @@ -1943,7 +1942,7 @@ isConstantFPBuildVectorOrConstantFP(N); } - void addCallSiteInfo(const SDNode *CallNode, CallSiteInfoImpl &&CallInfo) { + void addCallSiteInfo(const SDNode *CallNode, CallSiteInfo &&CallInfo) { SDCallSiteDbgInfo[CallNode].CSInfo = std::move(CallInfo); } diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp --- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp @@ -804,10 +804,10 @@ ParamSet &Params) { const MachineFunction *MF = CallMI->getMF(); const auto &CalleesMap = MF->getCallSitesInfo(); - auto CallFwdRegsInfo = CalleesMap.find(CallMI); + auto CallSiteInfo = CalleesMap.find(CallMI); // There is no information for the call instruction. - if (CallFwdRegsInfo == CalleesMap.end()) + if (CallSiteInfo == CalleesMap.end()) return; const MachineBasicBlock *MBB = CallMI->getParent(); @@ -821,7 +821,7 @@ DIExpression::get(MF->getFunction().getContext(), {}); // Add all the forwarding registers into the ForwardedRegWorklist. - for (const auto &ArgReg : CallFwdRegsInfo->second) { + for (const auto &ArgReg : CallSiteInfo->second.ArgRegPairs) { bool InsertedReg = ForwardedRegWorklist.insert({ArgReg.Reg, {{ArgReg.Reg, EmptyExpr}}}) .second; diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp --- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp +++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp @@ -401,14 +401,15 @@ Register Reg; if (parseNamedRegisterReference(PFS, Reg, ArgRegPair.Reg.Value, Error)) return error(Error, ArgRegPair.Reg.SourceRange); - CSInfo.emplace_back(Reg, ArgRegPair.ArgNo); + CSInfo.ArgRegPairs.emplace_back(Reg, ArgRegPair.ArgNo); } - if (TM.Options.EmitCallSiteInfo) - MF.addCallArgsForwardingRegs(&*CallI, std::move(CSInfo)); + if (TM.Options.EmitCallSiteInfo || TM.Options.EmitCallGraphSection) + MF.addCallSiteInfo(&*CallI, std::move(CSInfo)); } - if (YamlMF.CallSitesInfo.size() && !TM.Options.EmitCallSiteInfo) + if (YamlMF.CallSitesInfo.size() && + !(TM.Options.EmitCallSiteInfo || TM.Options.EmitCallGraphSection)) return error(Twine("Call site info provided but not used")); return false; } diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp --- a/llvm/lib/CodeGen/MIRPrinter.cpp +++ b/llvm/lib/CodeGen/MIRPrinter.cpp @@ -519,7 +519,7 @@ std::distance(CallI->getParent()->instr_begin(), CallI); YmlCS.CallLocation = CallLocation; // Construct call arguments and theirs forwarding register info. - for (auto ArgReg : CSInfo.second) { + for (auto ArgReg : CSInfo.second.ArgRegPairs) { yaml::CallSiteInfo::ArgRegPair YmlArgReg; YmlArgReg.ArgNo = ArgReg.ArgNo; printRegMIR(ArgReg.Reg, YmlArgReg.Reg, TRI); diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp --- a/llvm/lib/CodeGen/MachineFunction.cpp +++ b/llvm/lib/CodeGen/MachineFunction.cpp @@ -897,7 +897,7 @@ assert(MI->isCandidateForCallSiteEntry() && "Call site info refers only to call (MI) candidates"); - if (!Target.Options.EmitCallSiteInfo) + if (!Target.Options.EmitCallSiteInfo && !Target.Options.EmitCallGraphSection) return CallSitesInfo.end(); return CallSitesInfo.find(MI); } diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp --- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -887,8 +887,9 @@ } if (MI->isCandidateForCallSiteEntry() && - DAG->getTarget().Options.EmitCallSiteInfo) - MF.addCallArgsForwardingRegs(MI, DAG->getSDCallSiteInfo(Node)); + (DAG->getTarget().Options.EmitCallSiteInfo || + DAG->getTarget().Options.EmitCallGraphSection)) + MF.addCallSiteInfo(MI, DAG->getSDCallSiteInfo(Node)); if (DAG->getNoMergeSiteInfo(Node)) { MI->setFlag(MachineInstr::MIFlag::NoMerge); diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -5676,6 +5676,7 @@ bool &IsTailCall = CLI.IsTailCall; CallingConv::ID CallConv = CLI.CallConv; bool IsVarArg = CLI.IsVarArg; + const auto *CB = CLI.CB; MachineFunction &MF = DAG.getMachineFunction(); MachineFunction::CallSiteInfo CSInfo; @@ -5686,6 +5687,16 @@ bool IsSibCall = false; bool IsCalleeWin64 = Subtarget->isCallingConvWin64(CallConv); + // Set type id for call site info. + if (MF.getTarget().Options.EmitCallGraphSection && CB && + CB->isIndirectCall()) { + CSInfo.TypeId = MachineFunction::CallSiteInfo::extractNumericCGTypeId(*CB); + if (!CSInfo.TypeId) { + errs() << "warning: cannot find indirect call type id metadata for " + "call graph section\n"; + } + } + // Check callee args/returns for SVE registers and set calling convention // accordingly. if (CallConv == CallingConv::C || CallConv == CallingConv::Fast) { @@ -5953,15 +5964,16 @@ // Call site info is used for function's parameter entry value // tracking. For now we track only simple cases when parameter // is transferred through whole register. - llvm::erase_if(CSInfo, [&VA](MachineFunction::ArgRegPair ArgReg) { - return ArgReg.Reg == VA.getLocReg(); - }); + llvm::erase_if(CSInfo.ArgRegPairs, + [&VA](MachineFunction::ArgRegPair ArgReg) { + return ArgReg.Reg == VA.getLocReg(); + }); } else { RegsToPass.emplace_back(VA.getLocReg(), Arg); RegsUsed.insert(VA.getLocReg()); const TargetOptions &Options = DAG.getTarget().Options; if (Options.EmitCallSiteInfo) - CSInfo.emplace_back(VA.getLocReg(), i); + CSInfo.ArgRegPairs.emplace_back(VA.getLocReg(), i); } } else { assert(VA.isMemLoc()); diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -2287,6 +2287,7 @@ CallingConv::ID CallConv = CLI.CallConv; bool doesNotRet = CLI.DoesNotReturn; bool isVarArg = CLI.IsVarArg; + const auto *CB = CLI.CB; MachineFunction &MF = DAG.getMachineFunction(); ARMFunctionInfo *AFI = MF.getInfo(); @@ -2297,6 +2298,16 @@ bool isSibCall = false; bool PreferIndirect = false; + // Set type id for call site info. + if (MF.getTarget().Options.EmitCallGraphSection && CB && + CB->isIndirectCall()) { + CSInfo.TypeId = MachineFunction::CallSiteInfo::extractNumericCGTypeId(*CB); + if (!CSInfo.TypeId) { + errs() << "warning: cannot find indirect call type id metadata for call " + "graph section\n"; + } + } + // Determine whether this is a non-secure function call. if (CLI.CB && CLI.CB->getAttributes().hasFnAttribute("cmse_nonsecure_call")) isCmseNSCall = true; @@ -2493,7 +2504,7 @@ } const TargetOptions &Options = DAG.getTarget().Options; if (Options.EmitCallSiteInfo) - CSInfo.emplace_back(VA.getLocReg(), i); + CSInfo.ArgRegPairs.emplace_back(VA.getLocReg(), i); RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); } else if (isByVal) { assert(VA.isMemLoc()); diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp --- a/llvm/lib/Target/Mips/MipsISelLowering.cpp +++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp @@ -3152,6 +3152,7 @@ bool &IsTailCall = CLI.IsTailCall; CallingConv::ID CallConv = CLI.CallConv; bool IsVarArg = CLI.IsVarArg; + const auto *CB = CLI.CB; MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); @@ -3209,8 +3210,17 @@ // Get a count of how many bytes are to be pushed on the stack. unsigned NextStackOffset = CCInfo.getNextStackOffset(); - // Call site info for function parameters tracking. + // Call site info for function parameters tracking and call base type info. MachineFunction::CallSiteInfo CSInfo; + // Set type id for call site info. + if (MF.getTarget().Options.EmitCallGraphSection && CB && + CB->isIndirectCall()) { + CSInfo.TypeId = MachineFunction::CallSiteInfo::extractNumericCGTypeId(*CB); + if (!CSInfo.TypeId) { + errs() << "warning: cannot find indirect call type id metadata for call " + "graph section\n"; + } + } // Check if it's really possible to do a tail call. Restrict it to functions // that are part of this compilation unit. @@ -3349,8 +3359,8 @@ // Collect CSInfo about which register passes which parameter. const TargetOptions &Options = DAG.getTarget().Options; - if (Options.SupportsDebugEntryValues) - CSInfo.emplace_back(VA.getLocReg(), i); + if (Options.EmitCallSiteInfo && Options.SupportsDebugEntryValues) + CSInfo.ArgRegPairs.emplace_back(VA.getLocReg(), i); continue; } diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp --- a/llvm/lib/Target/X86/X86FastISel.cpp +++ b/llvm/lib/Target/X86/X86FastISel.cpp @@ -3579,6 +3579,19 @@ CLI.NumResultRegs = RVLocs.size(); CLI.Call = MIB; + // Add call site information (only call type id for call graph section). + if (TM.Options.EmitCallGraphSection && CB && CB->isIndirectCall()) { + auto *TypeId = MachineFunction::CallSiteInfo::extractNumericCGTypeId(*CB); + if (TypeId) { + MachineFunction::CallSiteInfo CSInfo; + CSInfo.TypeId = TypeId; + MF->addCallSiteInfo(CLI.Call, std::move(CSInfo)); + } else { + errs() << "warning: cannot find indirect call type id metadata for call " + "graph section\n"; + } + } + return true; } @@ -3969,6 +3982,8 @@ MO.setReg(IndexReg); } + if (MI->isCall()) + FuncInfo.MF->moveCallSiteInfo(MI, Result); Result->addMemOperand(*FuncInfo.MF, createMachineMemOperandFor(LI)); Result->cloneInstrSymbols(*FuncInfo.MF, *MI); MachineBasicBlock::iterator I(MI); diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -3937,6 +3937,16 @@ if (CallConv == CallingConv::X86_INTR) report_fatal_error("X86 interrupts may not be called directly"); + // Set type id for call site info. + if (MF.getTarget().Options.EmitCallGraphSection && CB && + CB->isIndirectCall()) { + CSInfo.TypeId = MachineFunction::CallSiteInfo::extractNumericCGTypeId(*CB); + if (!CSInfo.TypeId) { + errs() << "warning: cannot find indirect call type id metadata for call " + "graph section\n"; + } + } + bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall(); if (Subtarget.isPICStyleGOT() && !IsGuaranteeTCO && !IsMustTail) { // If we are using a GOT, disable tail calls to external symbols with @@ -4142,7 +4152,7 @@ RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); const TargetOptions &Options = DAG.getTarget().Options; if (Options.EmitCallSiteInfo) - CSInfo.emplace_back(VA.getLocReg(), I); + CSInfo.ArgRegPairs.emplace_back(VA.getLocReg(), I); if (isVarArg && IsWin64) { // Win64 ABI requires argument XMM reg to be copied to the corresponding // shadow reg if callee is a varargs function.