Index: llvm/lib/CodeGen/FixupStatepointCallerSaved.cpp =================================================================== --- llvm/lib/CodeGen/FixupStatepointCallerSaved.cpp +++ llvm/lib/CodeGen/FixupStatepointCallerSaved.cpp @@ -223,6 +223,7 @@ OpsToSpill.push_back(MI.getNumOperands()); unsigned CurOpIdx = 0; + SmallSet WrittenSlotIndices; for (unsigned I = 0; I < MI.getNumOperands(); ++I) { MachineOperand &MO = MI.getOperand(I); if (I == OpsToSpill[CurOpIdx]) { @@ -234,16 +235,29 @@ MIB.addFrameIndex(FI); MIB.addImm(0); ++CurOpIdx; + if (MO.isDef()) + WrittenSlotIndices.insert(FI); } else MIB.add(MO); } assert(CurOpIdx == (OpsToSpill.size() - 1) && "Not all operands processed"); + + // Note: This loop exists only to make a patch separably from D81647, once + // that lands, remove this loop. + for (MachineOperand &MO : MI.operands()) + if (MO.isReg() && !MO.isImplicit() && MO.isDef() && + RegToSlotIdx.count(MO.getReg())) + WrittenSlotIndices.insert(RegToSlotIdx[MO.getReg()]); + // Add mem operands. NewMI->setMemRefs(MF, MI.memoperands()); for (auto It : RegToSlotIdx) { int FrameIndex = It.second; auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); - auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, + MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad; + if (WrittenSlotIndices.count(FrameIndex)) + Flags |= MachineMemOperand::MOStore; + auto *MMO = MF.getMachineMemOperand(PtrInfo, Flags, getRegisterSize(TRI, It.first), MFI.getObjectAlign(FrameIndex)); NewMI->addMemOperand(MF, MMO); Index: llvm/test/CodeGen/X86/statepoint-vreg-fixup-csr.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/statepoint-vreg-fixup-csr.ll @@ -0,0 +1,84 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +; RUN: llc -max-registers-for-gc-values=5 -stop-after=fixup-statepoint-caller-saved -disable-peephole < %s | FileCheck %s +; -diable-peephole is a workaround for llvm bug 46916 + +target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-linux-gnu" + +declare void @foo() +declare void @consume(i8 addrspace(1)*) + +; Make sure that spilled gc operands get marked w/MOStore +define void @test(i8 addrspace(1)* %a, i8 addrspace(1)* %b, i8 addrspace(1)* %c, i8 addrspace(1)* %d, i8 addrspace(1)* %e, i8 addrspace(1)* %f, i8 addrspace(1)* %g, i8 addrspace(1)* %h) gc "statepoint-example" { + ; CHECK-LABEL: name: test + ; CHECK: bb.0.entry: + ; CHECK: liveins: $rcx, $rdi, $rdx, $rsi, $r8, $r9 + ; CHECK: renamable $r15 = COPY $r9 + ; CHECK: renamable $r13 = COPY $r8 + ; CHECK: renamable $rbx = COPY $rcx + ; CHECK: renamable $r12 = MOV64rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (load 8 from %fixed-stack.1, align 16) + ; CHECK: renamable $r14 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load 8 from %fixed-stack.0) + ; CHECK: MOV64mr %stack.1, 1, $noreg, 0, $noreg, killed renamable $rsi :: (store 8 into %stack.1) + ; CHECK: MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $rdx :: (store 8 into %stack.0) + ; CHECK: MOV64mr %stack.2, 1, $noreg, 0, $noreg, killed renamable $rdi :: (store 8 into %stack.2) + ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: renamable $r14, renamable $r12, renamable $r15, renamable $r13, dead renamable $rbx = STATEPOINT 0, 0, 0, @foo, 2, 0, 2, 0, 2, 0, killed renamable $r14, renamable $r14(tied-def 0), killed renamable $r12, renamable $r12(tied-def 1), killed renamable $r15, renamable $r15(tied-def 2), killed renamable $r13, renamable $r13(tied-def 3), killed renamable $rbx, renamable $rbx(tied-def 4), 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, 1, 8, %stack.1, 0, 1, 8, %stack.1, 0, 1, 8, %stack.2, 0, 1, 8, %stack.2, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0), (volatile load store 8 on %stack.1), (volatile load store 8 on %stack.2) + ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: renamable $rdi = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load 8 from %stack.2) + ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp + ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: renamable $rdi = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load 8 from %stack.1) + ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp + ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: renamable $rbx = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0) + ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: $rdi = COPY renamable $rbx + ; CHECK: CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp + ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: $rdi = COPY killed renamable $rbx + ; CHECK: CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp + ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: $rdi = COPY killed renamable $r13 + ; CHECK: CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp + ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: $rdi = COPY killed renamable $r15 + ; CHECK: CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp + ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: $rdi = COPY killed renamable $r12 + ; CHECK: CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp + ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: $rdi = COPY killed renamable $r14 + ; CHECK: CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp + ; CHECK: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + ; CHECK: RET 0 +entry: + %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i8 addrspace(1)* %a, i8 addrspace(1)* %b, i8 addrspace(1)* %c, i8 addrspace(1)* %d, i8 addrspace(1)* %e, i8 addrspace(1)* %f, i8 addrspace(1)* %g, i8 addrspace(1)* %h)] + %a1 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token, i32 0, i32 0) + call void @consume(i8 addrspace(1)* %a1) + %b1 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token, i32 1, i32 1) + call void @consume(i8 addrspace(1)* %b1) + %c1 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token, i32 2, i32 2) + call void @consume(i8 addrspace(1)* %c1) + %d1 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token, i32 3, i32 3) + call void @consume(i8 addrspace(1)* %c1) + %e1 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token, i32 4, i32 4) + call void @consume(i8 addrspace(1)* %e1) + %f1 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token, i32 5, i32 5) + call void @consume(i8 addrspace(1)* %f1) + %g1 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token, i32 6, i32 6) + call void @consume(i8 addrspace(1)* %g1) + %h1 = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token, i32 7, i32 7) + call void @consume(i8 addrspace(1)* %h1) + ret void +} + +declare token @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...) +declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...) +declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32)