Index: include/llvm/CodeGen/MIRYamlMapping.h =================================================================== --- include/llvm/CodeGen/MIRYamlMapping.h +++ include/llvm/CodeGen/MIRYamlMapping.h @@ -337,6 +337,61 @@ static const bool flow = true; }; +struct CallSiteInfo { + struct ArgRegPair { + uint32_t ArgNo; + StringValue Reg; + + bool operator==(const ArgRegPair &Other) const { + return ArgNo == Other.ArgNo && Reg == Other.Reg; + } + }; + + struct MachineInstrLoc { + unsigned BlockNum; + unsigned Offset; + + bool operator==(const MachineInstrLoc &Other) const { + return BlockNum == Other.BlockNum && Offset == Other.Offset; + } + }; + + MachineInstrLoc CallLocation; + std::vector ArgForwardingRegs; + + bool operator==(const CallSiteInfo &Other) const { + return CallLocation.BlockNum == Other.CallLocation.BlockNum && + CallLocation.Offset == Other.CallLocation.Offset; + } +}; + +template <> struct MappingTraits { + static void mapping(IO &YamlIO, CallSiteInfo::ArgRegPair &ArgReg) { + YamlIO.mapRequired("arg", ArgReg.ArgNo); + YamlIO.mapRequired("reg", ArgReg.Reg); + } + + static const bool flow = true; +}; +} +} + +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::CallSiteInfo::ArgRegPair) + +namespace llvm { +namespace yaml { + +template <> struct MappingTraits { + static void mapping(IO &YamlIO, CallSiteInfo &CSInfo) { + YamlIO.mapRequired("bb", CSInfo.CallLocation.BlockNum); + YamlIO.mapRequired("offset", CSInfo.CallLocation.Offset); + YamlIO.mapOptional("fwdArgRegs", CSInfo.ArgForwardingRegs, + std::vector()); + } + + static const bool flow = true; +}; + struct MachineConstantPoolValue { UnsignedValue ID; StringValue Value; @@ -391,6 +446,7 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::VirtualRegisterDefinition) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineStackObject) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::FixedMachineStackObject) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::CallSiteInfo) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineConstantPoolValue) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineJumpTable::Entry) @@ -519,6 +575,7 @@ std::vector StackObjects; std::vector Constants; /// Constant pool. std::unique_ptr MachineFuncInfo; + std::vector CallSitesInfo; MachineJumpTable JumpTableInfo; BlockStringValue Body; }; @@ -545,6 +602,8 @@ std::vector()); YamlIO.mapOptional("stack", MF.StackObjects, std::vector()); + YamlIO.mapOptional("callSites", MF.CallSitesInfo, + std::vector()); YamlIO.mapOptional("constants", MF.Constants, std::vector()); YamlIO.mapOptional("machineFunctionInfo", MF.MachineFuncInfo); Index: include/llvm/CodeGen/MachineFunction.h =================================================================== --- include/llvm/CodeGen/MachineFunction.h +++ include/llvm/CodeGen/MachineFunction.h @@ -30,6 +30,7 @@ #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Metadata.h" @@ -376,9 +377,17 @@ virtual void MF_HandleRemoval(MachineInstr &MI) = 0; }; + /// Vector of call argument and its forwarding register. + using CallSiteInfo = SmallVector, 1>; + using CallSiteInfoImpl = SmallVectorImpl>; + private: Delegate *TheDelegate = nullptr; + using CallSiteInfoMap = DenseMap; + /// Map a call instruction to call site arguments forwarding info. + CallSiteInfoMap CallSitesInfo; + // Callbacks for insertion and removal. void handleInsertion(MachineInstr &MI); void handleRemoval(MachineInstr &MI); @@ -931,6 +940,27 @@ const VariableDbgInfoMapTy &getVariableDbgInfo() const { return VariableDbgInfos; } + + void addCallArgsForwardingRegs(const MachineInstr *CallI, + CallSiteInfoImpl &&CallInfo) { + assert(CallI->isCall()); + if (STI->enableCallSiteInfo()) + CallSitesInfo[CallI] = std::move(CallInfo); + } + + const CallSiteInfoMap &getCallSitesInfo() const { + return CallSitesInfo; + } + + /// Update call sites info by deleting entry for /p Old call instruction. + /// If /p New is present then transfer /p Old call info to it. + void updateCallSiteInfo(const MachineInstr *Old, + const MachineInstr *New = nullptr) { + assert(Old->isCall() && (!New || New->isCall())); + if (New) + CallSitesInfo[New] = std::move(CallSitesInfo[Old]); + CallSitesInfo.erase(Old); + } }; //===--------------------------------------------------------------------===// Index: include/llvm/CodeGen/SelectionDAG.h =================================================================== --- include/llvm/CodeGen/SelectionDAG.h +++ include/llvm/CodeGen/SelectionDAG.h @@ -267,6 +267,10 @@ /// Tracks dbg_value and dbg_label information through SDISel. SDDbgInfo *DbgInfo; + using CallSiteInfo = MachineFunction::CallSiteInfo; + using CallSiteInfoImpl = MachineFunction::CallSiteInfoImpl; + DenseMap SDCallSiteInfo; + uint16_t NextPersistentId = 0; public: @@ -1619,6 +1623,17 @@ isConstantFPBuildVectorOrConstantFP(N); } + void addCallSiteInfo(const SDNode *CallNode, CallSiteInfoImpl &CallInfo) { + SDCallSiteInfo[CallNode] = std::move(CallInfo); + } + + CallSiteInfo getSDCallSiteInfo(const SDNode *CallNode) const { + auto I = SDCallSiteInfo.find(CallNode); + if (I != SDCallSiteInfo.end()) + return std::move(I->second); + return CallSiteInfo(); + } + private: void InsertNode(SDNode *N); bool RemoveNodeFromCSEMaps(SDNode *N); Index: include/llvm/CodeGen/TargetSubtargetInfo.h =================================================================== --- include/llvm/CodeGen/TargetSubtargetInfo.h +++ include/llvm/CodeGen/TargetSubtargetInfo.h @@ -211,6 +211,9 @@ /// True if the subtarget should run the indirectbr expansion pass. virtual bool enableIndirectBrExpand() const; + /// True if target supports generation of call site information. + virtual bool enableCallSiteInfo() const; + /// Override generic scheduling policy within a region. /// /// This is a convenient way for targets that don't provide any custom Index: lib/CodeGen/InlineSpiller.cpp =================================================================== --- lib/CodeGen/InlineSpiller.cpp +++ lib/CodeGen/InlineSpiller.cpp @@ -868,6 +868,8 @@ HSpiller.rmFromMergeableSpills(*MI, FI)) --NumSpills; LIS.ReplaceMachineInstrInMaps(*MI, *FoldMI); + if (MI->isCall()) + MI->getMF()->updateCallSiteInfo(MI, FoldMI); MI->eraseFromParent(); // Insert any new instructions other than FoldMI into the LIS maps. Index: lib/CodeGen/MIRParser/MIRParser.cpp =================================================================== --- lib/CodeGen/MIRParser/MIRParser.cpp +++ lib/CodeGen/MIRParser/MIRParser.cpp @@ -436,6 +436,22 @@ computeFunctionProperties(MF); + for (auto YamlCSInfo : YamlMF.CallSitesInfo) { + yaml::CallSiteInfo::MachineInstrLoc MILoc = YamlCSInfo.CallLocation; + auto CallB = std::next(MF.begin(), MILoc.BlockNum); + auto CallI = std::next(CallB->begin(), MILoc.Offset - 1); + assert(CallI->isCall() && "Call site info is related to call instruction"); + MachineFunction::CallSiteInfo CSInfo; + for (auto ArgRegPair : YamlCSInfo.ArgForwardingRegs) { + unsigned Reg = 0; + if (parseNamedRegisterReference(PFS, Reg, ArgRegPair.Reg.Value, + Error)) + return error(Error, ArgRegPair.Reg.SourceRange); + CSInfo.emplace_back(ArgRegPair.ArgNo, Reg); + } + MF.addCallArgsForwardingRegs(&*CallI, std::move(CSInfo)); + } + MF.getSubtarget().mirFileLoaded(MF); MF.verify(); Index: lib/CodeGen/MIRPrinter.cpp =================================================================== --- lib/CodeGen/MIRPrinter.cpp +++ lib/CodeGen/MIRPrinter.cpp @@ -128,6 +128,9 @@ const MachineJumpTableInfo &JTI); void convertStackObjects(yaml::MachineFunction &YMF, const MachineFunction &MF, ModuleSlotTracker &MST); + void convertCallSiteObjects(yaml::MachineFunction &YMF, + const MachineFunction &MF, + ModuleSlotTracker &MST); private: void initRegisterMaskIds(const MachineFunction &MF); @@ -211,6 +214,7 @@ MST.incorporateFunction(MF.getFunction()); convert(MST, YamlMF.FrameInfo, MF.getFrameInfo()); convertStackObjects(YamlMF, MF, MST); + convertCallSiteObjects(YamlMF, MF, MST); if (const auto *ConstantPool = MF.getConstantPool()) convert(YamlMF, *ConstantPool); if (const auto *JumpTableInfo = MF.getJumpTableInfo()) @@ -456,6 +460,41 @@ } } +void MIRPrinter::convertCallSiteObjects(yaml::MachineFunction &YMF, + const MachineFunction &MF, + ModuleSlotTracker &MST) { + const auto *TRI = MF.getSubtarget().getRegisterInfo(); + for (auto CSInfo : MF.getCallSitesInfo()) { + yaml::CallSiteInfo YmlCS; + yaml::CallSiteInfo::MachineInstrLoc CallLocation; + + // Prepare instruction position. + auto CallI = CSInfo.first->getReverseIterator(); + unsigned Offset = 0; + CallLocation.BlockNum = CallI->getParent()->getNumber(); + // Get call instruction offset from the beginning of block. + for (auto E = CallI->getParent()->rend(); CallI != E; CallI++, Offset++); + CallLocation.Offset = Offset; + YmlCS.CallLocation = CallLocation; + // Construct call arguments and theirs forwarding register info. + for (auto ArgReg : CSInfo.second) { + yaml::CallSiteInfo::ArgRegPair YmlArgReg; + YmlArgReg.ArgNo = ArgReg.first; + printRegMIR(ArgReg.second, YmlArgReg.Reg, TRI); + YmlCS.ArgForwardingRegs.emplace_back(YmlArgReg); + } + YMF.CallSitesInfo.push_back(YmlCS); + } + + // Sort call info by position of call instructions. + std::sort(YMF.CallSitesInfo.begin(), YMF.CallSitesInfo.end(), + [](yaml::CallSiteInfo A, yaml::CallSiteInfo B) { + if (A.CallLocation.BlockNum == B.CallLocation.BlockNum) + return A.CallLocation.Offset < B.CallLocation.Offset; + return A.CallLocation.BlockNum < B.CallLocation.BlockNum; + }); +} + void MIRPrinter::convert(yaml::MachineFunction &MF, const MachineConstantPool &ConstantPool) { unsigned ID = 0; Index: lib/CodeGen/MachineFunction.cpp =================================================================== --- lib/CodeGen/MachineFunction.cpp +++ lib/CodeGen/MachineFunction.cpp @@ -35,7 +35,6 @@ #include "llvm/CodeGen/TargetFrameLowering.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetRegisterInfo.h" -#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/CodeGen/WasmEHFuncInfo.h" #include "llvm/CodeGen/WinEHFuncInfo.h" #include "llvm/Config/llvm-config.h" @@ -362,6 +361,9 @@ /// ~MachineInstr() destructor must be empty. void MachineFunction::DeleteMachineInstr(MachineInstr *MI) { + if (MI->isCall(MachineInstr::IgnoreBundle)) + assert(CallSitesInfo.find(MI) == CallSitesInfo.end() && + "Call site info is not updated!"); // Strip it for parts. The operand array and the MI object itself are // independently recyclable. if (MI->Operands) Index: lib/CodeGen/MachineOutliner.cpp =================================================================== --- lib/CodeGen/MachineOutliner.cpp +++ lib/CodeGen/MachineOutliner.cpp @@ -1246,8 +1246,9 @@ if (MBB.getParent()->getProperties().hasProperty( MachineFunctionProperties::Property::TracksLiveness)) { // Helper lambda for adding implicit def operands to the call - // instruction. - auto CopyDefs = [&CallInst](MachineInstr &MI) { + // instruction. It also updates call site information for moved + // code. + auto CopyDefsAndUpdateCalls = [&CallInst](MachineInstr &MI) { for (MachineOperand &MOP : MI.operands()) { // Skip over anything that isn't a register. if (!MOP.isReg()) @@ -1259,13 +1260,16 @@ MOP.getReg(), true, /* isDef = true */ true /* isImp = true */)); } + if (MI.isCall()) + MI.getMF()->updateCallSiteInfo(&MI); }; // Copy over the defs in the outlined range. // First inst in outlined range <-- Anything that's defined in this // ... .. range has to be added as an // implicit Last inst in outlined range <-- def to the call - // instruction. - std::for_each(CallInst, std::next(EndIt), CopyDefs); + // instruction. Also remove call site information for outlined block + // of code. + std::for_each(CallInst, std::next(EndIt), CopyDefsAndUpdateCalls); } // Erase from the point after where the call was inserted up to, and Index: lib/CodeGen/MachineVerifier.cpp =================================================================== --- lib/CodeGen/MachineVerifier.cpp +++ lib/CodeGen/MachineVerifier.cpp @@ -2144,6 +2144,10 @@ verifyLiveVariables(); if (LiveInts) verifyLiveIntervals(); + + for (auto CSInfo : MF->getCallSitesInfo()) + if (!CSInfo.first->isCall()) + report("Call site info refrenceining instruction that is not call", MF); } void MachineVerifier::verifyLiveVariables() { Index: lib/CodeGen/PeepholeOptimizer.cpp =================================================================== --- lib/CodeGen/PeepholeOptimizer.cpp +++ lib/CodeGen/PeepholeOptimizer.cpp @@ -1777,6 +1777,8 @@ LocalMIs.erase(MI); LocalMIs.erase(DefMI); LocalMIs.insert(FoldMI); + if (MI->isCall()) + MI->getMF()->updateCallSiteInfo(MI, FoldMI); MI->eraseFromParent(); DefMI->eraseFromParent(); MRI->markUsesInDebugValueAsUndef(FoldedReg); Index: lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp =================================================================== --- lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -853,14 +853,20 @@ if (Before == After) return nullptr; + MachineInstr *MI; if (Before == BB->end()) { // There were no prior instructions; the new ones must start at the // beginning of the block. - return &Emitter.getBlock()->instr_front(); + MI = &Emitter.getBlock()->instr_front(); } else { // Return first instruction after the pre-existing instructions. - return &*std::next(Before); + MI = &*std::next(Before); } + + if (MI->isCall()) + MF.addCallArgsForwardingRegs(MI, DAG->getSDCallSiteInfo(Node)); + + return MI; }; // If this is the first BB, emit byval parameter dbg_value's. Index: lib/CodeGen/TargetInstrInfo.cpp =================================================================== --- lib/CodeGen/TargetInstrInfo.cpp +++ lib/CodeGen/TargetInstrInfo.cpp @@ -135,8 +135,15 @@ // Save off the debug loc before erasing the instruction. DebugLoc DL = Tail->getDebugLoc(); - // Remove all the dead instructions from the end of MBB. - MBB->erase(Tail, MBB->end()); + // Remove all the dead instructions from the end of MBB and update call site + // info. + while(Tail != MBB->end()) { + MachineInstr *MI = &*Tail; + if (MI->isCall()) + MBB->getParent()->updateCallSiteInfo(MI); + ++Tail; + MBB->erase(MI); + } // If MBB isn't immediately before MBB, insert a branch to it. if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(NewDest)) Index: lib/CodeGen/TargetSubtargetInfo.cpp =================================================================== --- lib/CodeGen/TargetSubtargetInfo.cpp +++ lib/CodeGen/TargetSubtargetInfo.cpp @@ -14,6 +14,9 @@ using namespace llvm; +static cl::opt EmitCallSiteInfo("emit-call-site-info", cl::Hidden, + cl::init(false)); + TargetSubtargetInfo::TargetSubtargetInfo( const Triple &TT, StringRef CPU, StringRef FS, ArrayRef PF, ArrayRef PD, @@ -33,6 +36,10 @@ return false; } +bool TargetSubtargetInfo::enableCallSiteInfo() const { + return EmitCallSiteInfo; +} + bool TargetSubtargetInfo::enableMachineScheduler() const { return false; } Index: lib/CodeGen/XRayInstrumentation.cpp =================================================================== --- lib/CodeGen/XRayInstrumentation.cpp +++ lib/CodeGen/XRayInstrumentation.cpp @@ -110,6 +110,8 @@ for (auto &MO : T.operands()) MIB.add(MO); Terminators.push_back(&T); + if (T.isCall()) + MF.updateCallSiteInfo(&T); } } } Index: lib/Target/X86/X86ExpandPseudo.cpp =================================================================== --- lib/Target/X86/X86ExpandPseudo.cpp +++ lib/Target/X86/X86ExpandPseudo.cpp @@ -270,6 +270,7 @@ MachineInstr &NewMI = *std::prev(MBBI); NewMI.copyImplicitOps(*MBBI->getParent()->getParent(), *MBBI); + MBB.getParent()->updateCallSiteInfo(&*MBBI, &NewMI); // Delete the pseudo instruction TCRETURN. MBB.erase(MBBI); Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -3586,6 +3586,8 @@ const Module *M = MF.getMMI().getModule(); Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch"); + MachineFunction::CallSiteInfo CSInfo; + if (CallConv == CallingConv::X86_INTR) report_fatal_error("X86 interrupts may not be called directly"); @@ -3781,6 +3783,8 @@ Subtarget); } else if (VA.isRegLoc()) { RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); + if (Subtarget.enableCallSiteInfo()) + CSInfo.emplace_back(I, VA.getLocReg()); if (isVarArg && IsWin64) { // Win64 ABI requires argument XMM reg to be copied to the corresponding // shadow reg if callee is a varargs function. @@ -4081,7 +4085,9 @@ // should be computed from returns not tail calls. Consider a void // function making a tail call to a function returning int. MF.getFrameInfo().setHasTailCall(); - return DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops); + SDValue Ret = DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops); + DAG.addCallSiteInfo(Ret.getNode(), CSInfo); + return Ret; } if (HasNoCfCheck && IsCFProtectionSupported) { @@ -4090,6 +4096,7 @@ Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops); } InFlag = Chain.getValue(1); + DAG.addCallSiteInfo(Chain.getNode(), CSInfo); // Create the CALLSEQ_END node. unsigned NumBytesForCalleeToPop; Index: test/CodeGen/X86/call-site-info-output.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/call-site-info-output.ll @@ -0,0 +1,35 @@ +; Test call site info MIR printer and parser.Parser assertions and machine +; verifier will check the rest; +; RUN: llc -emit-call-site-info %s -stop-after=expand-isel-pseudos -o %t.mir +; RUN: cat %t.mir | FileCheck %s +; CHECK: name: fn2 +; CHECK: callSites: +; CHECK-NEXT: bb: 0, offset: 14, fwdArgRegs: +; CHECK-NEXT: arg: 0, reg: '$edi' +; CHECK-NEXT: arg: 1, reg: '$esi' +; CHECK-NEXT: arg: 2, reg: '$edx' +; RUN: llc %t.mir -start-after=expand-isel-pseudos + +; ModuleID = 'test/CodeGen/X86/call-site-info-output.c' +source_filename = "test/CodeGen/X86/call-site-info-output.c" +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: noinline nounwind uwtable +define dso_local i64 @fn2(i32 %a, i32 %b, i32 %c) local_unnamed_addr #0 { +entry: + %call = tail call i32 (i32, i32, i32, ...) bitcast (i32 (...)* @fn1 to i32 (i32, i32, i32, ...)*)(i32 -50, i32 50, i32 -7) #2 + %add = mul i32 %a, 3 + %sub = sub i32 %add, %b + %add2 = add i32 %sub, %c + %conv4 = sext i32 %add2 to i64 + ret i64 %conv4 +} + +declare dso_local i32 @fn1(...) local_unnamed_addr #1 + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 9.0.0"}