diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp --- a/llvm/lib/CodeGen/InlineSpiller.cpp +++ b/llvm/lib/CodeGen/InlineSpiller.cpp @@ -810,6 +810,8 @@ bool WasCopy = MI->isCopy(); unsigned ImpReg = 0; + bool UntieRegs = MI->getOpcode() == TargetOpcode::STATEPOINT; + // Spill subregs if the target allows it. // We always want to spill subregs for stackmap/patchpoint pseudos. bool SpillSubRegs = TII.isSubregFoldable() || @@ -829,6 +831,9 @@ continue; } + if (UntieRegs && MO.isTied()) + MI->untieRegOperand(Idx); + if (!SpillSubRegs && MO.getSubReg()) return false; // We cannot fold a load instruction into a def. diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp --- a/llvm/lib/CodeGen/TargetInstrInfo.cpp +++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp @@ -471,6 +471,7 @@ ArrayRef Ops, int FrameIndex, const TargetInstrInfo &TII) { unsigned StartIdx = 0; + unsigned NumDefs = 0; switch (MI.getOpcode()) { case TargetOpcode::STACKMAP: { // StackMapLiveValues are foldable @@ -486,16 +487,28 @@ case TargetOpcode::STATEPOINT: { // For statepoints, fold deopt and gc arguments, but not call arguments. StartIdx = StatepointOpers(&MI).getVarIdx(); + NumDefs = MI.getNumDefs(); break; } default: llvm_unreachable("unexpected stackmap opcode"); } + unsigned DefToFoldIdx = MI.getNumOperands(); + // Return false if any operands requested for folding are not foldable (not // part of the stackmap's live values). for (unsigned Op : Ops) { - if (Op < StartIdx) + if (Op < NumDefs) + DefToFoldIdx = Op; + else if (Op < StartIdx) + return nullptr; + // When called from regalloc (InlineSpiller), operands must be untied, + // and regalloc will take care of (re)loading operand from memory. + // But when called from other places (e.g. peephole pass), + // we cannot fold operand which are tied - callers are unaware they + // need to reload destination register. + if (MI.getOperand(Op).isTied()) return nullptr; } @@ -505,11 +518,16 @@ // No need to fold return, the meta data, and function arguments for (unsigned i = 0; i < StartIdx; ++i) - MIB.add(MI.getOperand(i)); + if (i != DefToFoldIdx) + MIB.add(MI.getOperand(i)); - for (unsigned i = StartIdx; i < MI.getNumOperands(); ++i) { + for (unsigned i = StartIdx, e = MI.getNumOperands(); i < e; ++i) { MachineOperand &MO = MI.getOperand(i); + unsigned TiedTo = e; + (void)MI.isRegTiedToDefOperand(i, &TiedTo); + if (is_contained(Ops, i)) { + assert(TiedTo == e && "Cannot fold tied operands"); unsigned SpillSize; unsigned SpillOffset; // Compute the spill slot size and offset. @@ -523,9 +541,15 @@ MIB.addImm(SpillSize); MIB.addFrameIndex(FrameIndex); MIB.addImm(SpillOffset); - } - else + } else { MIB.add(MO); + if (TiedTo < e) { + assert(TiedTo < NumDefs && "Bad tied operand"); + if (TiedTo > DefToFoldIdx) + --TiedTo; + NewMI->tieOperands(TiedTo, NewMI->getNumOperands() - 1); + } + } } return NewMI; }