diff --git a/llvm/include/llvm/CodeGen/StackMaps.h b/llvm/include/llvm/CodeGen/StackMaps.h --- a/llvm/include/llvm/CodeGen/StackMaps.h +++ b/llvm/include/llvm/CodeGen/StackMaps.h @@ -225,6 +225,15 @@ return MI->getOperand(getNumDeoptArgsIdx()).getImm(); } + /// Get index of number of gc map entries. + unsigned getNumGcMapEntriesIdx(); + + /// Get index of number of gc allocas. + unsigned getNumAllocaIdx(); + + /// Get index of number of GC pointers. + unsigned getNumGCPtrIdx(); + /// Get index of first GC pointer operand of -1 if there are none. int getFirstGCPtrIdx(); diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1647,6 +1647,10 @@ } auto VerifyStackMapConstant = [&](unsigned Offset) { + if (Offset >= MI->getNumOperands()) { + report("stack map constant to STATEPOINT is out of range!", MI); + return; + } if (!MI->getOperand(Offset - 1).isImm() || MI->getOperand(Offset - 1).getImm() != StackMaps::ConstantOp || !MI->getOperand(Offset).isImm()) @@ -1655,6 +1659,9 @@ VerifyStackMapConstant(SO.getCCIdx()); VerifyStackMapConstant(SO.getFlagsIdx()); VerifyStackMapConstant(SO.getNumDeoptArgsIdx()); + VerifyStackMapConstant(SO.getNumGCPtrIdx()); + VerifyStackMapConstant(SO.getNumAllocaIdx()); + VerifyStackMapConstant(SO.getNumGcMapEntriesIdx()); // TODO: verify we have properly encoded deopt arguments } break; diff --git a/llvm/lib/CodeGen/StackMaps.cpp b/llvm/lib/CodeGen/StackMaps.cpp --- a/llvm/lib/CodeGen/StackMaps.cpp +++ b/llvm/lib/CodeGen/StackMaps.cpp @@ -91,41 +91,52 @@ return ScratchIdx; } -int StatepointOpers::getFirstGCPtrIdx() { - unsigned NumDeoptsIdx = getNumDeoptArgsIdx(); - unsigned NumDeoptArgs = MI->getOperand(NumDeoptsIdx).getImm(); +unsigned StatepointOpers::getNumGcMapEntriesIdx() { + // Take index of num of allocas and skip all allocas records. + unsigned CurIdx = getNumAllocaIdx(); + unsigned NumAllocas = getConstMetaVal(*MI, CurIdx - 1); + CurIdx++; + while (NumAllocas--) + CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx); + return CurIdx + 1; // skip +} - unsigned CurIdx = NumDeoptsIdx + 1; +unsigned StatepointOpers::getNumAllocaIdx() { + // Take index of num of gc ptrs and skip all gc ptr records. + unsigned CurIdx = getNumGCPtrIdx(); + unsigned NumGCPtrs = getConstMetaVal(*MI, CurIdx - 1); + CurIdx++; + while (NumGCPtrs--) + CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx); + return CurIdx + 1; // skip +} + +unsigned StatepointOpers::getNumGCPtrIdx() { + // Take index of num of deopt args and skip all deopt records. + unsigned CurIdx = getNumDeoptArgsIdx(); + unsigned NumDeoptArgs = getConstMetaVal(*MI, CurIdx - 1); + CurIdx++; while (NumDeoptArgs--) { CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx); } - ++CurIdx; // - unsigned NumGCPtrs = MI->getOperand(CurIdx).getImm(); + return CurIdx + 1; // skip +} + +int StatepointOpers::getFirstGCPtrIdx() { + unsigned NumGCPtrsIdx = getNumGCPtrIdx(); + unsigned NumGCPtrs = getConstMetaVal(*MI, NumGCPtrsIdx - 1); if (NumGCPtrs == 0) return -1; - ++CurIdx; // - assert(CurIdx < MI->getNumOperands() && "Index points past operand list"); - return (int)CurIdx; + ++NumGCPtrsIdx; // skip + assert(NumGCPtrsIdx < MI->getNumOperands()); + return (int)NumGCPtrsIdx; } unsigned StatepointOpers::getGCPointerMap( SmallVectorImpl> &GCMap) { - int FirstGCIdx = getFirstGCPtrIdx(); - if (FirstGCIdx == -1) - return 0; - unsigned NumGCPtr = getConstMetaVal(*MI, (unsigned)FirstGCIdx - 2); - unsigned CurIdx = (unsigned)FirstGCIdx; - while (NumGCPtr--) - CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx); - - unsigned NumAllocas = getConstMetaVal(*MI, CurIdx); - CurIdx += 2; - while (NumAllocas--) - CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx); - - assert(CurIdx < MI->getNumOperands()); - unsigned GCMapSize = getConstMetaVal(*MI, CurIdx); - CurIdx += 2; + unsigned CurIdx = getNumGcMapEntriesIdx(); + unsigned GCMapSize = getConstMetaVal(*MI, CurIdx - 1); + CurIdx++; for (unsigned N = 0; N < GCMapSize; ++N) { unsigned B = MI->getOperand(CurIdx++).getImm(); unsigned D = MI->getOperand(CurIdx++).getImm(); diff --git a/llvm/test/CodeGen/X86/non-value-mem-operand.mir b/llvm/test/CodeGen/X86/non-value-mem-operand.mir --- a/llvm/test/CodeGen/X86/non-value-mem-operand.mir +++ b/llvm/test/CodeGen/X86/non-value-mem-operand.mir @@ -206,7 +206,7 @@ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load 8 from %stack.5) MOV64mr $rsp, 1, $noreg, 48, $noreg, killed $rax :: (store 8 into %stack.3) $rax = MOV64ri @wibble - STATEPOINT 2882400000, 0, 0, killed $rax, 2, 0, 2, 0, 2, 30, 2, 1, 2, 0, 2, 99, 2, 0, 2, 12, 2, 0, 2, 10, 1, 8, $rsp, 24, 2, 10, 2, 0, 2, 10, 1, 8, $rsp, 16, 2, 10, 2, 4278124286, 2, 6, 2, 4278124286, 2, 7, 1, 8, $rsp, 8, 2, 99, 2, 0, 2, 7, 2, 4278124286, 2, 99, 2, 0, 2, 13, 1, 8, $rsp, 48, 2, 7, 2, 4278124286, 2, 99, 2, 0, csr_64, implicit-def $rsp :: (volatile load 8 from %stack.0), (volatile load 8 from %stack.1), (volatile load 8 from %stack.2), (volatile load 8 from %stack.3) + STATEPOINT 2882400000, 0, 0, killed $rax, 2, 0, 2, 0, 2, 30, 2, 1, 2, 0, 2, 99, 2, 0, 2, 12, 2, 0, 2, 10, 1, 8, $rsp, 24, 2, 10, 2, 0, 2, 10, 1, 8, $rsp, 16, 2, 10, 2, 4278124286, 2, 6, 2, 4278124286, 2, 7, 1, 8, $rsp, 8, 2, 99, 2, 0, 2, 7, 2, 4278124286, 2, 99, 2, 0, 2, 13, 1, 8, $rsp, 48, 2, 7, 2, 4278124286, 2, 99, 2, 0, 2, 0, 2, 0, 2, 0, csr_64, implicit-def $rsp :: (volatile load 8 from %stack.0), (volatile load 8 from %stack.1), (volatile load 8 from %stack.2), (volatile load 8 from %stack.3) $esi = XOR32rr undef $esi, undef $esi, implicit-def dead $eflags $r12 = IMPLICIT_DEF @@ -283,11 +283,11 @@ MOVSDmr $rsp, 1, $noreg, 8, $noreg, killed $xmm0 :: (store 8 into %stack.2) $rax = MOV64ri @wobble $edi = MOV32ri -121 - STATEPOINT 2882400000, 0, 1, killed $rax, $edi, 2, 0, 2, 0, 2, 38, 2, 1, 2, 0, 2, 270, 2, 4, 2, 12, 2, 0, 2, 11, 2, 4278124286, 2, 99, 2, 0, 2, 10, 1, 8, $rsp, 24, 2, 6, 2, 4278124286, 2, 99, 2, 0, 2, 99, 2, 0, 2, 10, 1, 8, $rsp, 16, 2, 10, 2, 4278124286, 2, 99, 2, 0, 2, 7, 1, 8, $rsp, 8, 2, 99, 2, 0, 2, 7, 2, 4278124286, 2, 99, 2, 0, 2, 13, 2, 4278124286, 2, 99, 2, 0, 2, 99, 2, 0, csr_64, implicit-def $rsp :: (volatile load 8 from %stack.0), (volatile load 8 from %stack.1), (volatile load 8 from %stack.2) + STATEPOINT 2882400000, 0, 1, killed $rax, $edi, 2, 0, 2, 0, 2, 38, 2, 1, 2, 0, 2, 270, 2, 4, 2, 12, 2, 0, 2, 11, 2, 4278124286, 2, 99, 2, 0, 2, 10, 1, 8, $rsp, 24, 2, 6, 2, 4278124286, 2, 99, 2, 0, 2, 99, 2, 0, 2, 10, 1, 8, $rsp, 16, 2, 10, 2, 4278124286, 2, 99, 2, 0, 2, 7, 1, 8, $rsp, 8, 2, 99, 2, 0, 2, 7, 2, 4278124286, 2, 99, 2, 0, 2, 13, 2, 4278124286, 2, 99, 2, 0, 2, 99, 2, 0, 2, 0, 2, 0, 2, 0, csr_64, implicit-def $rsp :: (volatile load 8 from %stack.0), (volatile load 8 from %stack.1), (volatile load 8 from %stack.2) bb.13.bb59: $rax = MOV64ri @wobble $edi = MOV32ri 8 - STATEPOINT 2882400000, 0, 1, killed $rax, $edi, 2, 0, 2, 0, 2, 38, 2, 1, 2, 0, 2, 123, 2, 4, 2, 12, 2, 0, 2, 13, 2, 0, 2, 99, 2, 4278124286, 2, 13, 2, 0, 2, 10, 2, 4278124286, 2, 99, 2, 4278124286, 2, 99, 2, 4278124286, 2, 99, 2, 4278124286, 2, 99, 2, 0, 2, 99, 2, 4278124286, 2, 99, 2, 4278124286, 2, 99, 2, 0, 2, 99, 2, 4278124286, 2, 99, 2, 0, 2, 13, 2, 0, 2, 99, 2, 4278124286, 2, 99, 2, 0, csr_64, implicit-def $rsp + STATEPOINT 2882400000, 0, 1, killed $rax, $edi, 2, 0, 2, 0, 2, 38, 2, 1, 2, 0, 2, 123, 2, 4, 2, 12, 2, 0, 2, 13, 2, 0, 2, 99, 2, 4278124286, 2, 13, 2, 0, 2, 10, 2, 4278124286, 2, 99, 2, 4278124286, 2, 99, 2, 4278124286, 2, 99, 2, 4278124286, 2, 99, 2, 0, 2, 99, 2, 4278124286, 2, 99, 2, 4278124286, 2, 99, 2, 0, 2, 99, 2, 4278124286, 2, 99, 2, 0, 2, 13, 2, 0, 2, 99, 2, 4278124286, 2, 99, 2, 0, 2, 0, 2, 0, 2, 0, csr_64, implicit-def $rsp ... diff --git a/llvm/test/CodeGen/X86/statepoint-fixup-call.mir b/llvm/test/CodeGen/X86/statepoint-fixup-call.mir --- a/llvm/test/CodeGen/X86/statepoint-fixup-call.mir +++ b/llvm/test/CodeGen/X86/statepoint-fixup-call.mir @@ -72,13 +72,13 @@ ; CHECK: liveins: $rdi ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp ; CHECK: MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed $rdi :: (store 8 into %stack.0) - ; CHECK: STATEPOINT 0, 0, 0, @foo, 2, 0, 2, 0, 2, 0, 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (load store 8 on %stack.0) + ; CHECK: STATEPOINT 0, 0, 0, @foo, 2, 0, 2, 0, 2, 0, 2, 1, 1, 8, %stack.0, 0, 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (load store 8 on %stack.0) ; CHECK: $rdi = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0) ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp ; CHECK: $rax = COPY killed renamable $rdi ; CHECK: RET 0, killed $rax ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - renamable $rdi = STATEPOINT 0, 0, 0, @foo, 2, 0, 2, 0, 2, 0, killed renamable $rdi, renamable $rdi(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp + renamable $rdi = STATEPOINT 0, 0, 0, @foo, 2, 0, 2, 0, 2, 0, 2, 1, killed renamable $rdi(tied-def 0), 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp $rax = COPY killed renamable $rdi RET 0, killed $rax diff --git a/llvm/test/CodeGen/X86/statepoint-fixup-copy-prop-neg.mir b/llvm/test/CodeGen/X86/statepoint-fixup-copy-prop-neg.mir --- a/llvm/test/CodeGen/X86/statepoint-fixup-copy-prop-neg.mir +++ b/llvm/test/CodeGen/X86/statepoint-fixup-copy-prop-neg.mir @@ -76,14 +76,16 @@ ; CHECK: liveins: $rcx, $rsi ; CHECK: renamable $rdi = COPY $rsi ; CHECK: MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $rcx :: (store 8 into %stack.0) + ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp ; CHECK: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rdi :: (store 8 into %stack.1) - ; CHECK: STATEPOINT 0, 0, 1, @foo, $rdi, 2, 0, 2, 0, 2, 1, 1, 8, %stack.1, 0, 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0), (load 8 from %stack.1) + ; CHECK: STATEPOINT 0, 0, 1, @foo, $rdi, 2, 0, 2, 0, 2, 1, 1, 8, %stack.1, 0, 2, 1, 1, 8, %stack.0, 0, 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0), (load 8 from %stack.1) + ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp ; CHECK: renamable $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0) ; CHECK: RET 0, $rax renamable $rdi = COPY $rsi MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $rcx :: (store 8 into %stack.0) ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp - STATEPOINT 0, 0, 1, @foo, $rdi, 2, 0, 2, 0, 2, 1, killed renamable $rdi, 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0) + STATEPOINT 0, 0, 1, @foo, $rdi, 2, 0, 2, 0, 2, 1, killed renamable $rdi, 2, 1, 1, 8, %stack.0, 0, 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0) ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp renamable $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0) RET 0, $rax