diff --git a/llvm/lib/CodeGen/FixupStatepointCallerSaved.cpp b/llvm/lib/CodeGen/FixupStatepointCallerSaved.cpp --- a/llvm/lib/CodeGen/FixupStatepointCallerSaved.cpp +++ b/llvm/lib/CodeGen/FixupStatepointCallerSaved.cpp @@ -46,8 +46,20 @@ cl::desc("Allow spill in spill slot of greater size than register size"), cl::Hidden); +static cl::opt PassGCPtrInCSR( + "fixup-allow-gcptr-in-csr", cl::Hidden, cl::init(false), + cl::desc("Allow passing GC Pointer arguments in callee saved registers")); + +// This is purely debugging option. +// It may be handy for investigating statepoint spilling issues. +static cl::opt MaxStatepointsWithRegs( + "fixup-max-csr-statepoints", cl::Hidden, + cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers")); + namespace { +class FrameIndexesCache; + class FixupStatepointCallerSaved : public MachineFunctionPass { public: static char ID; @@ -66,7 +78,12 @@ } bool runOnMachineFunction(MachineFunction &MF) override; + +private: + void collectGlobalFIs(MachineBasicBlock &BB, FrameIndexesCache &Cache, + const TargetRegisterInfo *TRI); }; + } // End anonymous namespace. char FixupStatepointCallerSaved::ID = 0; @@ -83,6 +100,105 @@ return TRI.getSpillSize(*RC); } +// Advance iterator to the next stack map entry +static MachineInstr::const_mop_iterator +advanceToNextStackMapElt(MachineInstr::const_mop_iterator MOI) { + if (MOI->isImm()) { + switch (MOI->getImm()) { + default: + llvm_unreachable("Unrecognized operand type."); + case StackMaps::DirectMemRefOp: + MOI += 2; // , + break; + case StackMaps::IndirectMemRefOp: + MOI += 3; // , , + break; + case StackMaps::ConstantOp: + MOI += 1; + break; + } + } + return ++MOI; +} + +// Return statepoint GC args as a set +static SmallSet collectGCRegs(MachineInstr &MI) { + StatepointOpers SO(&MI); + unsigned NumDeoptIdx = SO.getNumDeoptArgsIdx(); + unsigned NumDeoptArgs = MI.getOperand(NumDeoptIdx).getImm(); + MachineInstr::const_mop_iterator MOI(MI.operands_begin() + NumDeoptIdx + 1), + MOE(MI.operands_end()); + + // Skip deopt args + for (unsigned i = 0; i < NumDeoptArgs; ++i) + MOI = advanceToNextStackMapElt(MOI); + + SmallSet Result; + while (MOI != MOE) { + if (MOI->isReg() && !MOI->isImplicit()) + Result.insert(MOI->getReg()); + MOI = advanceToNextStackMapElt(MOI); + } + return Result; +} + +// Try to eliminate redundant copy to register which we're going to +// spill, i.e. try to change: +// X = COPY Y +// SPILL X +// to +// SPILL Y +// If there are no uses of X between copy and STATEPOINT, that COPY +// may be eliminated. +// Reg - register we're about to spill +// RI - On entry points to statepoint. +// On successful copy propagation set to new spill point. +// IsKill - set to true if COPY is Kill (there are no uses of Y) +// Returns either found source copy register or original one. +static Register performCopyPropagation(Register Reg, + MachineBasicBlock::iterator &RI, + bool &IsKill, const TargetInstrInfo &TII, + const TargetRegisterInfo &TRI) { + MachineBasicBlock *MBB = RI->getParent(); + MachineBasicBlock::reverse_iterator E = MBB->rend(); + MachineInstr *Def = nullptr, *Use = nullptr; + for (auto It = ++(RI.getReverse()); It != E; ++It) { + if (It->readsRegister(Reg, &TRI) && !Use) + Use = &*It; + if (It->modifiesRegister(Reg, &TRI)) { + Def = &*It; + break; + } + } + if (!Def) + return Reg; + + auto DestSrc = TII.isCopyInstr(*Def); + if (!DestSrc || DestSrc->Destination->getReg() != Reg) + return Reg; + + Register SrcReg = DestSrc->Source->getReg(); + + if (getRegisterSize(TRI, Reg) != getRegisterSize(TRI, SrcReg)) + return Reg; + + LLVM_DEBUG(dbgs() << "spillRegisters: perform copy propagation " + << printReg(Reg, &TRI) << " -> " << printReg(SrcReg, &TRI) + << "\n"); + + // Insert spill immediately after Def + RI = ++MachineBasicBlock::iterator(Def); + IsKill = DestSrc->Source->isKill(); + + // There are no uses of original register between COPY and STATEPOINT. + // There can't be any after STATEPOINT, so we can eliminate Def. + if (!Use) { + LLVM_DEBUG(dbgs() << "spillRegisters: removing dead copy " << *Def); + Def->eraseFromParent(); + } + return SrcReg; +} + namespace { // Cache used frame indexes during statepoint re-write to re-use them in // processing next statepoint instruction. @@ -105,6 +221,19 @@ // size will be increased. DenseMap Cache; + // Landing pad can be destination of several statepoints. Every register + // defined by such statepoints must be spilled to the same stack slot. + // This map keeps that information. + // NOTE: we assume that spill slot live ranges do not intersect. + using RegStatepointPair = std::pair; + DenseMap GlobalIndices; + + FrameIndexesPerSize &getCacheBucket(unsigned Size) { + // In FixupSCSExtendSlotSize mode the bucket with 0 index is used + // for all sizes. + return Cache[FixupSCSExtendSlotSize ? 0 : Size]; + } + public: FrameIndexesCache(MachineFrameInfo &MFI, const TargetRegisterInfo &TRI) : MFI(MFI), TRI(TRI) {} @@ -114,13 +243,40 @@ for (auto &It : Cache) It.second.Index = 0; } + + // Reset the FI cache and reserve slots assigned for shared landing pads + // processing. + void resetAndPreFill(const MachineInstr &CurMI) { + reset(); + // Since cache does not track what registers were assigned + // to what slots, we reserve all slots up to the highest one + // used by GlobalIndices[CurMI] + for (auto It : GlobalIndices) { + auto P = It.first; + auto FI = It.second; + if (P.second == &CurMI) { + auto &Line = getCacheBucket(getRegisterSize(TRI, P.first)); + auto Pos = llvm::find(Line.Slots, FI); + assert(Pos != Line.Slots.end() && "FI must have been allocated"); + Line.Index = (Pos - Line.Slots.begin()) + 1; + } + } + } + // Get frame index to spill the register. - int getFrameIndex(Register Reg) { + int getFrameIndex(Register Reg, MachineInstr *MI = nullptr) { + if (MI) { + auto It = GlobalIndices.find(std::make_pair(Reg, MI)); + if (It != GlobalIndices.end()) { + int FI = It->second; + LLVM_DEBUG(dbgs() << "Found global FI " << FI << " for register " + << printReg(Reg, &TRI) << " at " << *MI); + return FI; + } + } + unsigned Size = getRegisterSize(TRI, Reg); - // In FixupSCSExtendSlotSize mode the bucket with 0 index is used - // for all sizes. - unsigned Bucket = FixupSCSExtendSlotSize ? 0 : Size; - FrameIndexesPerSize &Line = Cache[Bucket]; + FrameIndexesPerSize &Line = getCacheBucket(Size); if (Line.Index < Line.Slots.size()) { int FI = Line.Slots[Line.Index++]; // If all sizes are kept together we probably need to extend the @@ -138,6 +294,7 @@ ++Line.Index; return FI; } + // Sort all registers to spill in descendent order. In the // FixupSCSExtendSlotSize mode it will minimize the total frame size. // In non FixupSCSExtendSlotSize mode we can skip this step. @@ -148,8 +305,32 @@ return getRegisterSize(TRI, A) > getRegisterSize(TRI, B); }); } + + // Record frame index to be used to spill register \p Reg at instr \p MI. + void addGlobalSpillSlot(Register Reg, MachineInstr *MI, int FI) { + auto P = std::make_pair(Reg, MI); + GlobalIndices.insert(std::make_pair(P, FI)); + } }; +// Check if we already inserted reload of register Reg from spill slot FI +// in basic block MBB. +// This can happen in EH pad block which is successor of several +// statepoints. +static bool hasRegReload(Register Reg, int FI, MachineBasicBlock *MBB, + const TargetInstrInfo *TII, + const TargetRegisterInfo *TRI) { + auto I = MBB->SkipPHIsLabelsAndDebug(MBB->begin()), E = MBB->end(); + int Dummy; + for (; I != E; ++I) { + if (TII->isLoadFromStackSlot(*I, Dummy) == Reg && Dummy == FI) + return true; + if (I->modifiesRegister(Reg, TRI) || I->readsRegister(Reg, TRI)) + return false; + } + return false; +} + // Describes the state of the current processing statepoint instruction. class StatepointState { private: @@ -163,26 +344,32 @@ const uint32_t *Mask; // Cache of frame indexes used on previous instruction processing. FrameIndexesCache &CacheFI; + bool AllowGCPtrInCSR; // Operands with physical registers requiring spilling. SmallVector OpsToSpill; // Set of register to spill. SmallVector RegsToSpill; + // Set of registers to reload after statepoint. + SmallVector RegsToReload; // Map Register to Frame Slot index. DenseMap RegToSlotIdx; public: StatepointState(MachineInstr &MI, const uint32_t *Mask, - FrameIndexesCache &CacheFI) + FrameIndexesCache &CacheFI, bool AllowGCPtrInCSR) : MI(MI), MF(*MI.getMF()), TRI(*MF.getSubtarget().getRegisterInfo()), TII(*MF.getSubtarget().getInstrInfo()), MFI(MF.getFrameInfo()), - Mask(Mask), CacheFI(CacheFI) {} + Mask(Mask), CacheFI(CacheFI), AllowGCPtrInCSR(AllowGCPtrInCSR) {} + // Return true if register is callee saved. bool isCalleeSaved(Register Reg) { return (Mask[Reg / 32] >> Reg % 32) & 1; } + // Iterates over statepoint meta args to find caller saver registers. // Also cache the size of found registers. // Returns true if caller save registers found. bool findRegistersToSpill() { SmallSet VisitedRegs; + SmallSet GCRegs = collectGCRegs(MI); for (unsigned Idx = StatepointOpers(&MI).getVarIdx(), EndIdx = MI.getNumOperands(); Idx < EndIdx; ++Idx) { @@ -191,8 +378,13 @@ continue; Register Reg = MO.getReg(); assert(Reg.isPhysical() && "Only physical regs are expected"); - if (isCalleeSaved(Reg)) + + if (isCalleeSaved(Reg) && (AllowGCPtrInCSR || !is_contained(GCRegs, Reg))) continue; + + LLVM_DEBUG(dbgs() << "Will spill " << printReg(Reg, &TRI) << " at index " + << Idx << "\n"); + if (VisitedRegs.insert(Reg).second) RegsToSpill.push_back(Reg); OpsToSpill.push_back(Idx); @@ -200,30 +392,123 @@ CacheFI.sortRegisters(RegsToSpill); return !RegsToSpill.empty(); } + // Spill all caller saved registers right before statepoint instruction. // Remember frame index where register is spilled. void spillRegisters() { for (Register Reg : RegsToSpill) { - int FI = CacheFI.getFrameIndex(Reg); + int FI = CacheFI.getFrameIndex(Reg, &MI); const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg); - TII.storeRegToStackSlot(*MI.getParent(), MI, Reg, true /*is_Kill*/, FI, - RC, &TRI); + NumSpilledRegisters++; RegToSlotIdx[Reg] = FI; + + LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, &TRI) << " to FI " << FI + << "\n"); + + // Perform trivial copy propagation + bool IsKill = true; + MachineBasicBlock::iterator InsertBefore(MI); + Reg = performCopyPropagation(Reg, InsertBefore, IsKill, TII, TRI); + + LLVM_DEBUG(dbgs() << "Insert spill before " << *InsertBefore); + TII.storeRegToStackSlot(*MI.getParent(), InsertBefore, Reg, IsKill, FI, + RC, &TRI); } } + + void insertReloadBefore(unsigned Reg, MachineBasicBlock::iterator It, + MachineBasicBlock *MBB) { + const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg); + int FI = RegToSlotIdx[Reg]; + if (It != MBB->end()) { + TII.loadRegFromStackSlot(*MBB, It, Reg, FI, RC, &TRI); + return; + } + + // To insert reload at the end of MBB, insert it before last instruction + // and then swap them. + assert(MBB->begin() != MBB->end() && "Empty block"); + --It; + TII.loadRegFromStackSlot(*MBB, It, Reg, FI, RC, &TRI); + MachineInstr *Reload = It->getPrevNode(); + int Dummy = 0; + assert(TII.isLoadFromStackSlot(*Reload, Dummy) == Reg); + assert(Dummy == FI); + MBB->remove(Reload); + MBB->insertAfter(It, Reload); + } + + // Insert reloads of (relocated) registers spilled in statepoint. + void insertReloads(MachineInstr *NewStatepoint) { + MachineBasicBlock *MBB = NewStatepoint->getParent(); + auto It = std::next(NewStatepoint->getIterator()); + + MachineBasicBlock *EHPad = nullptr; + MachineBasicBlock::iterator EHPadInsertPoint; + // Invoke statepoint must be last one in block. + bool Last = + std::none_of(It, MBB->end().getInstrIterator(), [](MachineInstr &I) { + return I.getOpcode() == TargetOpcode::STATEPOINT; + }); + if (Last) { + auto It = llvm::find_if(MBB->successors(), [](MachineBasicBlock *B) { + return B->isEHPad(); + }); + if (It != MBB->succ_end()) { + EHPad = *It; + EHPadInsertPoint = EHPad->SkipPHIsLabelsAndDebug(EHPad->begin()); + } + } + + for (auto Reg : RegsToReload) { + insertReloadBefore(Reg, It, MBB); + LLVM_DEBUG(dbgs() << "Reloading " << printReg(Reg, &TRI) << " from FI " + << RegToSlotIdx[Reg] << " after statepoint\n"); + + if (EHPad && !hasRegReload(Reg, RegToSlotIdx[Reg], EHPad, &TII, &TRI)) { + insertReloadBefore(Reg, EHPadInsertPoint, EHPad); + LLVM_DEBUG(dbgs() << "...also reload at EHPad " + << printMBBReference(*EHPad) << "\n"); + } + } + } + // Re-write statepoint machine instruction to replace caller saved operands // with indirect memory location (frame index). - void rewriteStatepoint() { + MachineInstr *rewriteStatepoint() { MachineInstr *NewMI = MF.CreateMachineInstr(TII.get(MI.getOpcode()), MI.getDebugLoc(), true); MachineInstrBuilder MIB(MF, NewMI); + unsigned NumOps = MI.getNumOperands(); + + // New indices for the remaining defs. + SmallVector NewIndices; + unsigned NumDefs = MI.getNumDefs(); + for (unsigned I = 0; I < NumDefs; ++I) { + MachineOperand &DefMO = MI.getOperand(I); + assert(DefMO.isReg() && DefMO.isDef() && "Expected Reg Def operand"); + Register Reg = DefMO.getReg(); + if (!AllowGCPtrInCSR) { + assert(is_contained(RegsToSpill, Reg)); + RegsToReload.push_back(Reg); + } else { + if (isCalleeSaved(Reg)) { + NewIndices.push_back(NewMI->getNumOperands()); + MIB.addReg(Reg, RegState::Define); + } else { + NewIndices.push_back(NumOps); + RegsToReload.push_back(Reg); + } + } + } + // Add End marker. OpsToSpill.push_back(MI.getNumOperands()); unsigned CurOpIdx = 0; - for (unsigned I = 0; I < MI.getNumOperands(); ++I) { + for (unsigned I = NumDefs; I < MI.getNumOperands(); ++I) { MachineOperand &MO = MI.getOperand(I); if (I == OpsToSpill[CurOpIdx]) { int FI = RegToSlotIdx[MO.getReg()]; @@ -234,8 +519,15 @@ MIB.addFrameIndex(FI); MIB.addImm(0); ++CurOpIdx; - } else + } else { MIB.add(MO); + unsigned OldDef; + if (AllowGCPtrInCSR && MI.isRegTiedToDefOperand(I, &OldDef)) { + assert(OldDef < NumDefs); + assert(NewIndices[OldDef] < NumOps); + MIB->tieOperands(NewIndices[OldDef], MIB->getNumOperands() - 1); + } + } } assert(CurOpIdx == (OpsToSpill.size() - 1) && "Not all operands processed"); // Add mem operands. @@ -248,9 +540,13 @@ MFI.getObjectAlign(FrameIndex)); NewMI->addMemOperand(MF, MMO); } + // Insert new statepoint and erase old one. MI.getParent()->insert(MI, NewMI); + + LLVM_DEBUG(dbgs() << "rewritten statepoint to : " << *NewMI << "\n"); MI.eraseFromParent(); + return NewMI; } }; @@ -265,27 +561,104 @@ : MF(MF), TRI(*MF.getSubtarget().getRegisterInfo()), CacheFI(MF.getFrameInfo(), TRI) {} - bool process(MachineInstr &MI) { + StatepointProcessor(MachineFunction &MF, FrameIndexesCache &Cache) + : MF(MF), TRI(*MF.getSubtarget().getRegisterInfo()), CacheFI(Cache) {} + + bool process(MachineInstr &MI, bool AllowGCPtrInCSR) { StatepointOpers SO(&MI); uint64_t Flags = SO.getFlags(); // Do nothing for LiveIn, it supports all registers. if (Flags & (uint64_t)StatepointFlags::DeoptLiveIn) return false; + LLVM_DEBUG(dbgs() << "\nMBB " << MI.getParent()->getNumber() << " " + << MI.getParent()->getName() << " : process statepoint " + << MI); CallingConv::ID CC = SO.getCallingConv(); const uint32_t *Mask = TRI.getCallPreservedMask(MF, CC); - CacheFI.reset(); - StatepointState SS(MI, Mask, CacheFI); + CacheFI.resetAndPreFill(MI); + StatepointState SS(MI, Mask, CacheFI, AllowGCPtrInCSR); if (!SS.findRegistersToSpill()) return false; SS.spillRegisters(); - SS.rewriteStatepoint(); + auto *NewStatepoint = SS.rewriteStatepoint(); + SS.insertReloads(NewStatepoint); return true; } }; } // namespace +// Return live out definition of Reg in MBB or null +static MachineInstr *findLiveOutDef(Register Reg, MachineBasicBlock *MBB, + const TargetRegisterInfo *TRI) { + bool StatepointSeen = false; + for (auto I = MBB->rbegin(), E = MBB->rend(); I != E; ++I) { + // Special case for statepoint because we're looking specifically + // for explicit defs, not any implicit effects like regmask or register + // implicit def. + if (I->getOpcode() == TargetOpcode::STATEPOINT) { + int DefIdx = I->findRegisterDefOperandIdx(Reg, false, false, TRI); + if (DefIdx >= 0 && (unsigned)DefIdx < I->getNumDefs()) { + assert(!StatepointSeen && "Found gc value live across statepoint"); + return &*I; + } + StatepointSeen = true; + } else if (I->modifiesRegister(Reg, TRI)) { + return &*I; + } + } + return nullptr; +} + +// For EH pad block with multiple predecessors check if its live-in +// registers are defined by statepoints in preds. If so, assign same +// spill slot for each register at each statepoint. +// NOTE: It works only if all reaching definitions of register are statepoints, +// otherwise we cannot insert reload into EH pad and must insert multiple +// reloads on edges. +void FixupStatepointCallerSaved::collectGlobalFIs( + MachineBasicBlock &BB, FrameIndexesCache &Cache, + const TargetRegisterInfo *TRI) { + if (!BB.isEHPad() || BB.livein_empty() || BB.pred_size() == 1) + return; + SmallVector Preds(BB.predecessors()); + auto isStatepoint = [](MachineInstr *I) { + return I && I->getOpcode() == TargetOpcode::STATEPOINT; + }; + + // Resetting the cache allows us to reuse stack slots between + // different 'statepoint sets' (a set of statepoints reaching + // same EH Pad). This works under assumption that we allocate + // these 'global' spill slots before starting to process + // individual statepoints. + Cache.reset(); + + for (const auto &LI : BB.liveins()) { + Register Reg = LI.PhysReg; + SmallVector RegDefs; + for (auto *B : Preds) + RegDefs.push_back(findLiveOutDef(Reg, B, TRI)); + if (llvm::all_of(RegDefs, isStatepoint)) { + int FI = Cache.getFrameIndex(Reg); + for (auto *Def : RegDefs) { + Cache.addGlobalSpillSlot(Reg, Def, FI); + LLVM_DEBUG(dbgs() << "EH Pad bb." << BB.getNumber() << ": reserving FI " + << FI << " to spill register " << printReg(Reg, TRI) + << " at statepoint in bb." + << Def->getParent()->getNumber() << "\n"); + } + } else { + // That spilling stuff is all-or-nothing: either all defining instructions + // are statepoints (and we can spill to the same slot) or none of them are + // statepoints (so we do not need any reloads). Otherwise we're in + // trouble. + assert(llvm::none_of(RegDefs, isStatepoint) && + "Cannot safely reload register"); + } + } +} + bool FixupStatepointCallerSaved::runOnMachineFunction(MachineFunction &MF) { if (skipFunction(MF.getFunction())) return false; @@ -294,18 +667,30 @@ if (!F.hasGC()) return false; + const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); + FrameIndexesCache FICache(MF.getFrameInfo(), *TRI); + SmallVector Statepoints; - for (MachineBasicBlock &BB : MF) + for (MachineBasicBlock &BB : MF) { + collectGlobalFIs(BB, FICache, TRI); for (MachineInstr &I : BB) if (I.getOpcode() == TargetOpcode::STATEPOINT) Statepoints.push_back(&I); + } if (Statepoints.empty()) return false; bool Changed = false; - StatepointProcessor SPP(MF); - for (MachineInstr *I : Statepoints) - Changed |= SPP.process(*I); + StatepointProcessor SPP(MF, FICache); + unsigned NumStatepoints = 0; + bool AllowGCPtrInCSR = PassGCPtrInCSR; + for (MachineInstr *I : Statepoints) { + ++NumStatepoints; + if (MaxStatepointsWithRegs.getNumOccurrences() && + NumStatepoints >= MaxStatepointsWithRegs) + AllowGCPtrInCSR = false; + Changed |= SPP.process(*I, AllowGCPtrInCSR); + } return Changed; } diff --git a/llvm/test/CodeGen/X86/statepoint-fixup-call.mir b/llvm/test/CodeGen/X86/statepoint-fixup-call.mir new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/statepoint-fixup-call.mir @@ -0,0 +1,86 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -o - %s -fixup-allow-gcptr-in-csr=false -run-pass fixup-statepoint-caller-saved | FileCheck %s +--- | + ; ModuleID = 'test/CodeGen/X86/statepoint-fixup-call.ll' + source_filename = "test/CodeGen/X86/statepoint-fixup-call.ll" + target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + target triple = "x86_64-pc-linux-gnu" + + declare void @foo() + + define i8 addrspace(1)* @test_one(i8 addrspace(1)* %p) gc "statepoint-example" { + entry: + %token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* %p) ] + %p2 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %token, i32 0, i32 0) ; (%p, %p) + ret i8 addrspace(1)* %p2 + } + + declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 immarg, i32 immarg, void ()*, i32 immarg, i32 immarg, ...) + + ; Function Attrs: nounwind readonly + declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32 immarg, i32 immarg) #0 + + ; Function Attrs: nounwind + declare void @llvm.stackprotector(i8*, i8**) #1 + + attributes #0 = { nounwind readonly } + attributes #1 = { nounwind } + +... +--- +name: test_one +alignment: 16 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +failedISel: false +tracksRegLiveness: true +hasWinCFI: false +registers: [] +liveins: + - { reg: '$rdi', virtual-reg: '' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 1 + adjustsStack: false + hasCalls: true + stackProtector: '' + maxCallFrameSize: 4294967295 + cvBytesOfCalleeSavedRegisters: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false + localFrameSize: 0 + savePoint: '' + restorePoint: '' +fixedStack: [] +stack: [] +callSites: [] +constants: [] +machineFunctionInfo: {} +body: | + bb.0.entry: + liveins: $rdi + + ; CHECK-LABEL: name: test_one + ; CHECK: liveins: $rdi + ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed $rdi :: (store 8 into %stack.0) + ; CHECK: STATEPOINT 0, 0, 0, @foo, 2, 0, 2, 0, 2, 0, 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (load 8 from %stack.0) + ; CHECK: $rdi = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0) + ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: $rax = COPY killed renamable $rdi + ; CHECK: RET 0, killed $rax + ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + renamable $rdi = STATEPOINT 0, 0, 0, @foo, 2, 0, 2, 0, 2, 0, killed renamable $rdi, renamable $rdi(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp + ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + $rax = COPY killed renamable $rdi + RET 0, killed $rax + +... diff --git a/llvm/test/CodeGen/X86/statepoint-fixup-invoke.mir b/llvm/test/CodeGen/X86/statepoint-fixup-invoke.mir new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/statepoint-fixup-invoke.mir @@ -0,0 +1,145 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -o - %s -fixup-allow-gcptr-in-csr=false -run-pass fixup-statepoint-caller-saved | FileCheck %s + +--- | + ; ModuleID = 'test/CodeGen/X86/statepoint-fixup-invoke.mir' + source_filename = "test/CodeGen/X86/statepoint-fixup-invoke.mir" + target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + target triple = "x86_64-pc-linux-gnu" + + declare void @some_call(i64 addrspace(1)*) + + declare i32 @personality_function() + + define i64 addrspace(1)* @test_basic(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" personality i32 ()* @personality_function { + entry: + %0 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 0) [ "gc-live"(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1), "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ] + to label %invoke_safepoint_normal_dest unwind label %exceptional_return + + invoke_safepoint_normal_dest: ; preds = %entry + %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %0, i32 0, i32 0) ; (%obj, %obj) + %obj1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %0, i32 1, i32 1) ; (%obj1, %obj1) + br label %normal_return + + normal_return: ; preds = %invoke_safepoint_normal_dest + ret i64 addrspace(1)* %obj.relocated + + exceptional_return: ; preds = %entry + %landing_pad = landingpad token + cleanup + %obj.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 0, i32 0) ; (%obj, %obj) + %obj1.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 1, i32 1) ; (%obj1, %obj1) + ret i64 addrspace(1)* %obj1.relocated1 + } + + declare token @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 immarg, i32 immarg, void (i64 addrspace(1)*)*, i32 immarg, i32 immarg, ...) + + ; Function Attrs: nounwind readonly + declare i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token, i32 immarg, i32 immarg) #0 + + ; Function Attrs: nounwind + declare void @llvm.stackprotector(i8*, i8**) #1 + + attributes #0 = { nounwind readonly } + attributes #1 = { nounwind } + +... +--- +name: test_basic +alignment: 16 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +failedISel: false +tracksRegLiveness: true +hasWinCFI: false +registers: [] +liveins: + - { reg: '$rdi', virtual-reg: '' } + - { reg: '$rsi', virtual-reg: '' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 1 + adjustsStack: false + hasCalls: true + stackProtector: '' + maxCallFrameSize: 4294967295 + cvBytesOfCalleeSavedRegisters: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false + localFrameSize: 0 + savePoint: '' + restorePoint: '' +fixedStack: [] +stack: [] +callSites: [] +constants: [] +machineFunctionInfo: {} +body: | + ; CHECK-LABEL: name: test_basic + ; CHECK: bb.0.entry: + ; CHECK: successors: %bb.1(0x7ffff800), %bb.3(0x00000800) + ; CHECK: liveins: $rdi, $rsi + ; CHECK: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rsi :: (store 8 into %stack.0) + ; CHECK: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rdi :: (store 8 into %stack.1) + ; CHECK: EH_LABEL + ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: STATEPOINT 0, 0, 1, @some_call, $rdi, 2, 0, 2, 0, 2, 5, 2, 0, 2, -1, 2, 0, 2, 0, 2, 0, 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, 1, 8, %stack.1, 0, 1, 8, %stack.1, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (load 8 from %stack.1), (load 8 from %stack.0) + ; CHECK: $r14 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0) + ; CHECK: $rbx = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load 8 from %stack.1) + ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: EH_LABEL + ; CHECK: JMP_1 %bb.1 + ; CHECK: bb.1.invoke_safepoint_normal_dest: + ; CHECK: successors: %bb.2(0x80000000) + ; CHECK: liveins: $rbx + ; CHECK: bb.2.normal_return: + ; CHECK: liveins: $rbx + ; CHECK: $rax = COPY killed renamable $rbx + ; CHECK: RET 0, $rax + ; CHECK: bb.3.exceptional_return (landing-pad): + ; CHECK: liveins: $rax, $rdx, $r14 + ; CHECK: EH_LABEL + ; CHECK: $r14 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0) + ; CHECK: $rbx = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load 8 from %stack.1) + ; CHECK: $rax = COPY killed renamable $r14 + ; CHECK: RET 0, $rax + bb.0.entry: + successors: %bb.1(0x7ffff800), %bb.3(0x00000800) + liveins: $rdi, $rsi + + renamable $r14 = COPY $rsi + renamable $rbx = COPY $rdi + EH_LABEL + ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + renamable $r14, renamable $rbx = STATEPOINT 0, 0, 1, @some_call, $rdi, 2, 0, 2, 0, 2, 5, 2, 0, 2, -1, 2, 0, 2, 0, 2, 0, killed renamable $r14, renamable $r14(tied-def 0), killed renamable $rbx, renamable $rbx(tied-def 1), csr_64, implicit-def $rsp, implicit-def $ssp + ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + EH_LABEL + JMP_1 %bb.1 + + bb.1.invoke_safepoint_normal_dest: + successors: %bb.2(0x80000000) + liveins: $rbx + + + bb.2.normal_return: + liveins: $rbx + + $rax = COPY killed renamable $rbx + RET 0, $rax + + bb.3.exceptional_return (landing-pad): + liveins: $rax, $rdx, $r14 + + EH_LABEL + $rax = COPY killed renamable $r14 + RET 0, $rax + +... diff --git a/llvm/test/CodeGen/X86/statepoint-fixup-shared-ehpad.mir b/llvm/test/CodeGen/X86/statepoint-fixup-shared-ehpad.mir new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/statepoint-fixup-shared-ehpad.mir @@ -0,0 +1,184 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -o - %s -fixup-allow-gcptr-in-csr=false -run-pass fixup-statepoint-caller-saved | FileCheck %s + +--- | + ; ModuleID = 'statepoint-fixup-shared-ehpad.ll' + source_filename = "statepoint-fixup-shared-ehpad.ll" + target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + target triple = "x86_64-pc-linux-gnu" + + declare void @foo() + + declare void @bar() + + declare i32 @personality_function() + + define i8 addrspace(1)* @test_one(i32 %a, i8 addrspace(1)* %p, i8 addrspace(1)* %q) gc "statepoint-example" personality i32 ()* @personality_function { + entry: + %cmp = icmp eq i32 %a, 0 + br i1 %cmp, label %zero, label %nonzero + + zero: ; preds = %entry + %token = invoke token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* %p, i8 addrspace(1)* %q) ] + to label %normal_dest_a unwind label %exceptional_return_a + + nonzero: ; preds = %entry + %token2 = invoke token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @bar, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* %p, i8 addrspace(1)* %q) ] + to label %normal_dest_b unwind label %exceptional_return_b + + normal_dest_a: ; preds = %zero + %p2 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %token, i32 0, i32 0) ; (%p, %p) + ret i8 addrspace(1)* %p2 + + exceptional_return_a: ; preds = %zero + %landing_pad = landingpad token + cleanup + %q2 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %landing_pad, i32 1, i32 1) ; (%q, %q) + ret i8 addrspace(1)* %q2 + + normal_dest_b: ; preds = %nonzero + %p3 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %token2, i32 0, i32 0) ; (%p, %p) + ret i8 addrspace(1)* %p3 + + exceptional_return_b: ; preds = %nonzero + %landing_pad2 = landingpad token + cleanup + %q3 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %landing_pad2, i32 1, i32 1) ; (%q, %q) + ret i8 addrspace(1)* %q3 + } + + declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 immarg, i32 immarg, void ()*, i32 immarg, i32 immarg, ...) + + ; Function Attrs: nounwind readonly + declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32 immarg, i32 immarg) #0 + + attributes #0 = { nounwind readonly } + attributes #1 = { nounwind } + +... +--- +name: test_one +alignment: 16 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +failedISel: false +tracksRegLiveness: true +hasWinCFI: false +registers: [] +liveins: + - { reg: '$edi', virtual-reg: '' } + - { reg: '$rsi', virtual-reg: '' } + - { reg: '$rdx', virtual-reg: '' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 1 + adjustsStack: false + hasCalls: true + stackProtector: '' + maxCallFrameSize: 4294967295 + cvBytesOfCalleeSavedRegisters: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false + localFrameSize: 0 + savePoint: '' + restorePoint: '' +fixedStack: [] +stack: [] +callSites: [] +constants: [] +machineFunctionInfo: {} +body: | + ; CHECK-LABEL: name: test_one + ; CHECK: bb.1: + ; CHECK: successors: %bb.3(0x40000000), %bb.4(0x40000000) + ; CHECK: liveins: $rbx, $r14 + ; CHECK: EH_LABEL + ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: MOV64mr %stack.1, 1, $noreg, 0, $noreg, killed $rbx :: (store 8 into %stack.1) + ; CHECK: MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed $r14 :: (store 8 into %stack.0) + ; CHECK: STATEPOINT 0, 0, 0, @foo, 2, 0, 2, 0, 2, 0, 1, 8, %stack.1, 0, 1, 8, %stack.1, 0, 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (load 8 from %stack.1), (load 8 from %stack.0) + ; CHECK: $rbx = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load 8 from %stack.1) + ; CHECK: $r14 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0) + ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: EH_LABEL + ; CHECK: JMP_1 %bb.3 + ; CHECK: bb.2: + ; CHECK: successors: %bb.5(0x40000000), %bb.4(0x40000000) + ; CHECK: liveins: $rbx, $r14 + ; CHECK: EH_LABEL + ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: MOV64mr %stack.1, 1, $noreg, 0, $noreg, killed $rbx :: (store 8 into %stack.1) + ; CHECK: MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed $r14 :: (store 8 into %stack.0) + ; CHECK: STATEPOINT 0, 0, 0, @bar, 2, 0, 2, 0, 2, 0, 1, 8, %stack.1, 0, 1, 8, %stack.1, 0, 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (load 8 from %stack.1), (load 8 from %stack.0) + ; CHECK: $rbx = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load 8 from %stack.1) + ; CHECK: $r14 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0) + ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: EH_LABEL + ; CHECK: JMP_1 %bb.5 + ; CHECK: bb.4 (landing-pad): + ; CHECK: liveins: $rax, $rdx, $r14 + ; CHECK: EH_LABEL + ; CHECK: $rbx = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load 8 from %stack.1) + ; CHECK: $r14 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0) + ; CHECK: $rax = COPY killed renamable $r14 + ; CHECK: RET 0, $rax + bb.0: + successors: %bb.1, %bb.2 + liveins: $edi, $rdx, $rsi + + renamable $r14 = COPY $rdx + renamable $rbx = COPY $rsi + TEST32rr killed renamable $edi, renamable $edi, implicit-def $eflags + JCC_1 %bb.2, 5, implicit killed $eflags + JMP_1 %bb.1 + + bb.1: + successors: %bb.3, %bb.4 + liveins: $rbx, $r14 + + EH_LABEL + ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + renamable $rbx, renamable $r14 = STATEPOINT 0, 0, 0, @foo, 2, 0, 2, 0, 2, 0, killed renamable $rbx, renamable $rbx(tied-def 0), killed renamable $r14, renamable $r14(tied-def 1), csr_64, implicit-def $rsp, implicit-def $ssp + ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + EH_LABEL + JMP_1 %bb.3 + + bb.2: + successors: %bb.5, %bb.4 + liveins: $rbx, $r14 + + EH_LABEL + ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + renamable $rbx, renamable $r14 = STATEPOINT 0, 0, 0, @bar, 2, 0, 2, 0, 2, 0, killed renamable $rbx, renamable $rbx(tied-def 0), killed renamable $r14, renamable $r14(tied-def 1), csr_64, implicit-def $rsp, implicit-def $ssp + ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + EH_LABEL + JMP_1 %bb.5 + + bb.3: + liveins: $rbx + + $rax = COPY killed renamable $rbx + RET 0, $rax + + bb.4 (landing-pad): + liveins: $rax, $rdx, $r14 + + EH_LABEL + $rax = COPY killed renamable $r14 + RET 0, $rax + + bb.5: + liveins: $rbx + + $rax = COPY killed renamable $rbx + RET 0, $rax + +... diff --git a/llvm/test/CodeGen/X86/statepoint-vreg.mir b/llvm/test/CodeGen/X86/statepoint-vreg.mir --- a/llvm/test/CodeGen/X86/statepoint-vreg.mir +++ b/llvm/test/CodeGen/X86/statepoint-vreg.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -# RUN: llc -o - %s -start-after=finalize-isel | FileCheck %s +# RUN: llc -o - %s -fixup-allow-gcptr-in-csr=true -start-after=finalize-isel | FileCheck %s --- | ; ModuleID = 'test/CodeGen/X86/statepoint-vreg.ll'