MIR Statepoint refactoring. Part 5: Handle non-local relocates in ISEL.

[dantrushin]

Summary

When gc.relocate is in different basic block from statepoint,
corresponding statepoint result must be exported to register.
Default LLVM mechanism does not work here, because on IR level
statepoint does not produce relocated values.
We have to manually export statepoint results into register and
copy it back at gc.relocate location. We also need a dedicated
map for that because we cannot use FunctionLowering.ValueMap.
Depends On D81648

[dantrushin]

Remove accidently added test option

[dantrushin]

Another attempt

[reames]

JFYI, I've posted (and now landed after review) the subset of this patch which is mostly NFC here: https://reviews.llvm.org/D84692

I plan to reimplement the functional change (i.e. the rest of it) shortly.

I ended up duplicating Denis's work here by accident, but we at least arrived at a common solution (modulo style), so that's a good sign.

Marking as Request Changes to indicate need to either close or rebase if there's anything left.

llvm/include/llvm/CodeGen/FunctionLoweringInfo.h

public:
   /// Keep track of frame indices allocated for statepoints as they could be
   /// used across basic block boundaries (e.g. for an invoke).  For each
   /// gc.statepoint instruction, maps uniqued llvm IR values to the slots they
   /// were spilled in.  If a value is mapped to None it means we visited the
   /// value but didn't spill it (because it was a constant, for instance).
   using StatepointSpillMapTy = DenseMap<const Value *, Optional<int>>;
   DenseMap<const Instruction *, StatepointSpillMapTy> StatepointSpillMaps;
 
+  /// For each statepoint keep virtual registers its result values has
+  /// been exported to. This is similar to ValueMap, but for statepoint
+  /// results which has no existing IR Value.
+  using StatepointRegMap = DenseMap<const Value *, Register>;
+  DenseMap<const Instruction *, StatepointRegMap> StatepointRegs;
+
   /// StaticAllocaMap - Keep track of frame indices for fixed sized allocas in
   /// the entry block.  This allows the allocas to be efficiently referenced
   /// anywhere in the function.
   DenseMap<const AllocaInst*, int> StaticAllocaMap;
 
   /// ByValArgFrameIndexMap - Keep track of frame indices for byval arguments.
   DenseMap<const Argument*, int> ByValArgFrameIndexMap;
 

llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp

 /// Lower a single value incoming to a statepoint node.  This value can be
 /// either a deopt value or a gc value, the handling is the same.  We special
 /// case constants and allocas, then fall back to spilling if required.
 static void
 lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot,
                              SmallVectorImpl<SDValue> &Ops,
                              SmallVectorImpl<MachineMemOperand *> &MemRefs,
                              SelectionDAGBuilder &Builder) {
-  
   if (willLowerDirectly(Incoming)) {
     if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
       // This handles allocas as arguments to the statepoint (this is only
       // really meaningful for a deopt value.  For GC, we'd be trying to
       // relocate the address of the alloca itself?)
       assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
              "Incoming value is a frame index!");
       Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
                                                     Builder.getFrameIndexTy()));
 
       auto &MF = Builder.DAG.getMachineFunction();
       auto *MMO = getMachineMemOperand(MF, *FI);
       MemRefs.push_back(MMO);
       return;
     }
 
     assert(Incoming.getValueType().getSizeInBits() <= 64);
-    
     if (Incoming.isUndef()) {
       // Put an easily recognized constant that's unlikely to be a valid
       // value so that uses of undef by the consumer of the stackmap is
       // easily recognized. This is legal since the compiler is always
       // allowed to chose an arbitrary value for undef.
       pushStackMapConstant(Ops, Builder, 0xFEFEFEFE);
       return;
     }
if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
                            C->getValueAPF().bitcastToAPInt().getZExtValue());
       return;
     }
 
     llvm_unreachable("unhandled direct lowering case");
   }
 
 
-
   if (!RequireSpillSlot) {
     // If this value is live in (not live-on-return, or live-through), we can
     // treat it the same way patchpoint treats it's "live in" values.  We'll
     // end up folding some of these into stack references, but they'll be
     // handled by the register allocator.  Note that we do not have the notion
     // of a late use so these values might be placed in registers which are
     // clobbered by the call.  This is fine for live-in. For live-through
     // fix-up pass should be executed to force spilling of such registers.
#endif
   // potentially allow live-through values in callee saved registers.
   const bool LiveInDeopt =
     SI.StatepointFlags & (uint64_t)StatepointFlags::DeoptLiveIn;
 
   // Decide which deriver pointers will go on VRegs
   const unsigned MaxTiedRegs = 15; // Max  number of tied regs MI can have.
   unsigned MaxVRegPtrs =
       std::min(MaxTiedRegs, MaxRegistersForGCPointers.getValue());
-  // Use old spill scheme for cross-block relocates.
-  if (SI.StatepointInstr) {
-    const BasicBlock *BB = SI.StatepointInstr->getParent();
-    bool NonLocalReloc =
-        llvm::any_of(SI.GCRelocates, [BB](const GCRelocateInst *R) {
-          return R->getParent() != BB;
-        });
-    if (NonLocalReloc)
-      MaxVRegPtrs = 0;
-  }
 
   LLVM_DEBUG(dbgs() << "Desiding how to lower GC Pointers:\n");
   unsigned CurNumVRegs = 0;
   for (const Value *P : SI.Ptrs) {
     if (LowerAsVReg.size() == MaxVRegPtrs)
       break;
     SDValue PtrSD = Builder.getValue(P);
     if (willLowerDirectly(PtrSD) || P->getType()->isVectorTy()) {
SI.GCTransitionArgs = ArrayRef<const Use>(I.gc_transition_args_begin(),
                                             I.gc_transition_args_end());
 
   SI.StatepointFlags = I.getFlags();
   SI.NumPatchBytes = I.getNumPatchBytes();
   SI.EHPadBB = EHPadBB;
 
   SDValue ReturnValue = LowerAsSTATEPOINT(SI);
 
+  // If relocated value will be used in different basic
+  // block, we need to export them manually. Default exporting mechanism
+  // will not work here because it is based on IR Value types, and
+  // IR statepoint has different type than the actual call or relocates.
+  // It means that by default llvm will create export register of the wrong
+  // type (always i32 - TokenTy - in our case). So instead we need to create
+  // export registers manually.
+  const BasicBlock *BB = I.getParent();
+  auto &DPtrMap = StatepointLowering.DerivedPtrMap;
+  auto &StatepointRegMap = FuncInfo.StatepointRegs[SI.StatepointInstr];
+  for (const auto *Reloc : SI.GCRelocates) {
+    if (Reloc->getParent() == BB)
+      continue;
+
+    const Value *DerivedPtr = Reloc->getDerivedPtr();
+    if (!DPtrMap.count(DerivedPtr))
+      continue; // not lowered through VReg
+    if (StatepointRegMap.count(DerivedPtr))
+      continue;
+
+    SDValue Res = DPtrMap[DerivedPtr];
+    Type *Ty = DerivedPtr->getType();
+    Register Reg = FuncInfo.CreateRegs(Ty);
+    RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
+                     DAG.getDataLayout(), Reg, Ty, None);
+    SDValue Chain = DAG.getEntryNode();
+
+    RFV.getCopyToRegs(Res, DAG, getCurSDLoc(), Chain, nullptr, DerivedPtr);
+    PendingExports.push_back(Chain);
+    StatepointRegMap[DerivedPtr] = Reg;
+  }
+
   // Export the result value if needed
   const GCResultInst *GCResult = I.getGCResult();
   Type *RetTy = I.getActualReturnType();
 
   if (RetTy->isVoidTy() || !GCResult) {
     // The return value is not needed, just generate a poison value. 
     setValue(&I, DAG.getIntPtrConstant(-1, getCurSDLoc()));
     return;
#endif
   // manually.
   // TODO: To eliminate this problem we can remove gc.result intrinsics
   //       completely and make statepoint call to return a tuple.
   unsigned Reg = FuncInfo.CreateRegs(RetTy);
   RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
                    DAG.getDataLayout(), Reg, RetTy,
                    I.getCallingConv());
   SDValue Chain = DAG.getEntryNode();
-  
   RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
   PendingExports.push_back(Chain);
   FuncInfo.ValueMap[&I] = Reg;
 }
 
 void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl(
     const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB,
     bool VarArgDisallowed, bool ForceVoidReturnTy) {
void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
   // Statepoint is in different basic block so we should have stored call
   // result in a virtual register.
   // We can not use default getValue() functionality to copy value from this
   // register because statepoint and actual call return types can be
   // different, and getValue() will use CopyFromReg of the wrong type,
   // which is always i32 in our case.
   Type *RetTy = SI->getActualReturnType();
   SDValue CopyFromReg = getCopyFromRegs(SI, RetTy);
-  
   assert(CopyFromReg.getNode());
   setValue(&CI, CopyFromReg);
 }
 
 void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
   const BasicBlock *StatepointBB = Relocate.getStatepoint()->getParent();
 #ifndef NDEBUG
   // Consistency check
#endif
   auto &DPtrMap = StatepointLowering.DerivedPtrMap;
   auto It = DPtrMap.find(DerivedPtr);
   if (It != DPtrMap.end()) {
     setValue(&Relocate, It->second);
     assert(Relocate.getParent() == StatepointBB && "unexpected DPtrMap entry");
     return;
   }
 
+  auto &StatepointRegs = FuncInfo.StatepointRegs[Relocate.getStatepoint()];
+  if (StatepointRegs.count(DerivedPtr)) {
+    // Statepoint is in different basic block. Default getValue() mechanism
+    // does not work here, so we need create CopyFromRegs manually.
+    // See comment in LowerStatepoint for details.
+    assert(StatepointBB != Relocate.getParent() && "local gc.relocate");
+    Register Reg = StatepointRegs[DerivedPtr];
+    assert(Reg > 0 && "invalid register");
+    RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
+                     DAG.getDataLayout(), Reg, DerivedPtr->getType(),
+                     None); // This is not an ABI copy.
+    SDValue Chain = DAG.getEntryNode();
+    SDValue Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain,
+                                         nullptr, DerivedPtr);
+    setValue(&Relocate, Result);
+    return;
+  }
+
   auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.getStatepoint()];
   auto SlotIt = SpillMap.find(DerivedPtr);
   assert(SlotIt != SpillMap.end() && "Relocating not lowered gc value");
   Optional<int> DerivedPtrLocation = SlotIt->second;
 
   // We didn't need to spill these special cases (constants and allocas).
   // See the handling in spillIncomingValueForStatepoint for detail.
   if (!DerivedPtrLocation) {

llvm/test/CodeGen/X86/statepoint-vreg.ll

 ; This run is to demonstrate what MIR SSA looks like.
 ; RUN: llc -max-registers-for-gc-values=4 -stop-after finalize-isel < %s | FileCheck --check-prefix=CHECK-VREG %s
 ; This run is to demonstrate register allocator work.
 ; RUN: llc -max-registers-for-gc-values=4 -stop-after virtregrewriter < %s | FileCheck --check-prefix=CHECK-PREG %s
 ; This run is to demonstrate resulting assembly/stackmaps.
 ; NOTE: When D81647 is landed this run line will need to be adjusted!
 ; RUN: llc -max-registers-for-gc-values=4 < %s | FileCheck --check-prefix=CHECK-ASM %s
 
 target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 declare i1 @return_i1()
 declare void @func()
+declare void @"some_call"(i64 addrspace(1)*)
 declare void @consume(i32 addrspace(1)*)
 declare void @consume2(i32 addrspace(1)*, i32 addrspace(1)*)
 declare void @consume5(i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*)
 declare void @use1(i32 addrspace(1)*, i8 addrspace(1)*)
+declare i32 @"personality_function"()
 
 ; test most simple relocate
 define i1 @test_relocate(i32 addrspace(1)* %a) gc "statepoint-example" {
 ; CHECK-VREG-LABEL: name:            test_relocate
 ; CHECK-VREG:    %0:gr64 = COPY $rdi
 ; CHECK-VREG:    %1:gr64 = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al
 ; CHECK-VREG:    %2:gr8 = COPY $al
 ; CHECK-VREG:    $rdi = COPY %1

 ; CHECK-ASM-NEXT: retq
 entry:
   %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a)]
   %rel1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
   %res1 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)
   call void @consume(i32 addrspace(1)* %rel1)
   ret i1 %res1
 }
+
 ; test pointer variables intermixed with pointer constants
 define void @test_mixed(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) gc "statepoint-example" {
 ; CHECK-VREG-LABEL: name:            test_mixed
 ; CHECK-VREG:    %2:gr64 = COPY $rdx
 ; CHECK-VREG:    %1:gr64 = COPY $rsi
 ; CHECK-VREG:    %0:gr64 = COPY $rdi
 ; CHECK-VREG:    %3:gr64, %4:gr64, %5:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, %2, %2(tied-def 0), 2, 0, 2, 0, %1, %1(tied-def 1), %0, %0(tied-def 2), csr_64
 ; CHECK-VREG:    %6:gr32 = MOV32r0 implicit-def dead $eflags
%tok = tail call token (i64, i32, void ()*, i32, i32, ...)
       @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["deopt" (i32 addrspace(1)* %ptr, i32 undef), "gc-live" (i32 addrspace(1)* %ptr, i8 addrspace(1)* %ptr2)]
   %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 0, i32 0)
   %b = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok, i32 1, i32 1)
   call void @use1(i32 addrspace(1)* %a, i8 addrspace(1)* %b)
   ret void
 }
 
 ;
-; Cross-basicblock relocates are handled with spilling for now.
-; No need to check post-RA output
+; test non-local relocates
 define i1 @test_cross_bb(i32 addrspace(1)* %a, i1 %external_cond) gc "statepoint-example" {
 ; CHECK-VREG-LABEL: name:            test_cross_bb
 ; CHECK-VREG:  bb.0.entry:
-; CHECK-VREG:         %1:gr32 = COPY $esi
-; CHECK-VREG-NEXT:    %0:gr64 = COPY $rdi
-; CHECK-VREG-NEXT:    %3:gr8 = COPY %1.sub_8bit
-; CHECK-VREG-NEXT:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 8 into %stack.0)
-; CHECK-VREG-NEXT:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG-NEXT:    STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al :: (volatile load store 8 on %stack.0)
-; CHECK-VREG-NEXT:    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG-NEXT:    %4:gr8 = COPY $al
-; CHECK-VREG-NEXT:    %2:gr8 = COPY %4
-; CHECK-VREG-NEXT:    TEST8ri killed %3, 1, implicit-def $eflags
-; CHECK-VREG-NEXT:    JCC_1 %bb.2, 4, implicit $eflags
-; CHECK-VREG-NEXT:    JMP_1 %bb.1
-; CHECK-VREG:       bb.1.left:
-; CHECK-VREG-NEXT:    %6:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0)
-; CHECK-VREG-NEXT:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG-NEXT:    $rdi = COPY %6
-; CHECK-VREG-NEXT:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
-; CHECK-VREG-NEXT:    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG-NEXT:    $al = COPY %2
-; CHECK-VREG-NEXT:    RET 0, $al
-; CHECK-VREG:       bb.2.right:
-; CHECK-VREG-NEXT:    %5:gr8 = MOV8ri 1
-; CHECK-VREG-NEXT:    $al = COPY %5
-; CHECK-VREG-NEXT:    RET 0, $al
+; CHECK-VREG:    %1:gr32 = COPY $esi
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    %4:gr8 = COPY %1.sub_8bit
+; CHECK-VREG:    %2:gr64 = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al
+; CHECK-VREG:    %5:gr8 = COPY $al
+; CHECK-VREG:    %3:gr8 = COPY %5
+; CHECK-VREG:    TEST8ri killed %4, 1, implicit-def $eflags
+; CHECK-VREG:    JCC_1 %bb.2, 4, implicit $eflags
+; CHECK-VREG:    JMP_1 %bb.1
+; CHECK-VREG:  bb.1.left:
+; CHECK-VREG:    $rdi = COPY %2
+; CHECK-VREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    $al = COPY %3
+; CHECK-VREG:    RET 0, $al
+; CHECK-VREG:  bb.2.right:
+; CHECK-VREG:    %6:gr8 = MOV8ri 1
+; CHECK-VREG:    $al = COPY %6
+; CHECK-VREG:    RET 0, $al
 
 entry:
   %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a)]
   br i1 %external_cond, label %left, label %right
 
 left:
   %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
   %call2 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)
entry:
   %rel2 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 1, i32 1)
   %rel3 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 2, i32 2)
   %rel4 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 3, i32 3)
   %rel5 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 4, i32 4)
   call void @consume5(i32 addrspace(1)* %rel1, i32 addrspace(1)* %rel2, i32 addrspace(1)* %rel3, i32 addrspace(1)* %rel4, i32 addrspace(1)* %rel5)
   ret void
 }
 
+define i64 addrspace(1)* @test_basic_invoke(i64 addrspace(1)* %obj,
+                                            i64 addrspace(1)* %obj1)
+; CHECK-VREG-LABEL: name:            test_basic_invoke
+; CHECK-VREG:  bb.0.entry:
+; CHECK-VREG:    %2:gr64 = COPY $rsi
+; CHECK-VREG:    %1:gr64 = COPY $rdi
+; CHECK-VREG:    EH_LABEL <mcsymbol >
+; CHECK-VREG:    $rdi = COPY %1
+; CHECK-VREG:    %3:gr64, %4:gr64 = STATEPOINT 0, 0, 1, @some_call, $rdi, 2, 0, 2, 0, 2, 5, 2, 0, 2, -1, 2, 0, 2, 0, 2, 0, %2, %2(tied-def 0), %1, %1(tied-def 1), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    EH_LABEL <mcsymbol >
+; CHECK-VREG:    JMP_1 %bb.1
+; CHECK-VREG:  bb.1.invoke_safepoint_normal_dest:
+; CHECK-VREG:    %0:gr64 = COPY %4
+; CHECK-VREG:  bb.2.normal_return:
+; CHECK-VREG:    $rax = COPY %0
+; CHECK-VREG:    RET 0, $rax
+; CHECK-VREG:  bb.3.exceptional_return (landing-pad):
+; CHECK-VREG:    liveins: $rax, $rdx
+; CHECK-VREG:    EH_LABEL <mcsymbol >
+; CHECK-VREG:    %6:gr64 = COPY killed $rdx
+; CHECK-VREG:    %5:gr64 = COPY killed $rax
+; CHECK-VREG:    $rax = COPY %3
+; CHECK-VREG:    RET 0, $rax
+
+; CHECK-PREG-LABEL: name:            test_basic_invoke
+; CHECK-PREG:  bb.0.entry:
+; CHECK-PREG:    renamable $r14 = COPY $rsi
+; CHECK-PREG:    renamable $rbx = COPY $rdi
+; CHECK-PREG:    EH_LABEL <mcsymbol >
+; CHECK-PREG:    $rdi = COPY renamable $rbx
+; CHECK-PREG:    renamable $r14, renamable $rbx = STATEPOINT 0, 0, 1, @some_call, $rdi, 2, 0, 2, 0, 2, 5, 2, 0, 2, -1, 2, 0, 2, 0, 2, 0, killed renamable $r14, renamable $r14(tied-def 0), killed renamable $rbx, renamable $rbx(tied-def 1), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-PREG:    EH_LABEL <mcsymbol >
+; CHECK-PREG:    JMP_1 %bb.1
+; CHECK-PREG:  bb.1.invoke_safepoint_normal_dest:
+; CHECK-PREG:    successors: %bb.2(0x80000000)
+; CHECK-PREG:  bb.2.normal_return:
+; CHECK-PREG:    $rax = COPY killed renamable $rbx
+; CHECK-PREG:    RET 0, $rax
+; CHECK-PREG:  bb.3.exceptional_return (landing-pad):
+; CHECK-PREG:    EH_LABEL <mcsymbol >
+; CHECK-PREG:    $rax = COPY killed renamable $r14
+; CHECK-PREG:    RET 0, $rax
+gc "statepoint-example" personality i32 ()* @"personality_function" {
+entry:
+  %0 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 0) ["gc-live" (i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1), "deopt" (i32 0, i32 -1, i32 0, i32 0, i32 0)]
+          to label %invoke_safepoint_normal_dest unwind label %exceptional_return
+
+invoke_safepoint_normal_dest:
+  %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %0, i32 0, i32 0)
+  %obj1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %0, i32 1, i32 1)
+  br label %normal_return
+
+normal_return:
+  ret i64 addrspace(1)* %obj.relocated
+
+exceptional_return:
+  %landing_pad = landingpad token
+          cleanup
+  %obj.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 0, i32 0)
+  %obj1.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 1, i32 1)
+  ret i64 addrspace(1)* %obj1.relocated1
+}
+
+define i64 addrspace(1)* @test_invoke_same_val(i1 %cond, i64 addrspace(1)* %val1, i64 addrspace(1)* %val2, i64 addrspace(1)* %val3)
+; CHECK-VREG-LABEL: name:            test_invoke_same_val
+; CHECK-VREG:  bb.0.entry:
+; CHECK-VREG:    %9:gr64 = COPY $rcx
+; CHECK-VREG:    %8:gr64 = COPY $rdx
+; CHECK-VREG:    %7:gr64 = COPY $rsi
+; CHECK-VREG:    %6:gr32 = COPY $edi
+; CHECK-VREG:    %10:gr8 = COPY %6.sub_8bit
+; CHECK-VREG:    TEST8ri %10, 1, implicit-def $eflags
+; CHECK-VREG:    JCC_1 %bb.3, 4, implicit $eflags
+; CHECK-VREG:    JMP_1 %bb.1
+; CHECK-VREG:  bb.1.left:
+; CHECK-VREG:    EH_LABEL <mcsymbol >
+; CHECK-VREG:    $rdi = COPY %7
+; CHECK-VREG:    %15:gr64, %16:gr64 = STATEPOINT 0, 0, 1, @some_call, $rdi, 2, 0, 2, 0, 2, 0, %8, %8(tied-def 0), %7, %7(tied-def 1), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    EH_LABEL <mcsymbol >
+; CHECK-VREG:    JMP_1 %bb.2
+; CHECK-VREG:  bb.2.left.relocs:
+; CHECK-VREG:    %0:gr64 = COPY %16
+; CHECK-VREG:    %1:gr64 = COPY %15
+; CHECK-VREG:    JMP_1 %bb.5
+; CHECK-VREG:  bb.3.right:
+; CHECK-VREG:    EH_LABEL <mcsymbol >
+; CHECK-VREG:    $rdi = COPY %7
+; CHECK-VREG:    %11:gr64, %12:gr64 = STATEPOINT 0, 0, 1, @some_call, $rdi, 2, 0, 2, 0, 2, 0, %9, %9(tied-def 0), %8, %8(tied-def 1), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    EH_LABEL <mcsymbol >
+; CHECK-VREG:    JMP_1 %bb.4
+; CHECK-VREG:  bb.4.right.relocs:
+; CHECK-VREG:    %2:gr64 = COPY %12
+; CHECK-VREG:    %3:gr64 = COPY %11
+; CHECK-VREG:  bb.5.normal_return:
+; CHECK-VREG:    %4:gr64 = PHI %3, %bb.4, %0, %bb.2
+; CHECK-VREG:    %5:gr64 = PHI %2, %bb.4, %1, %bb.2
+; CHECK-VREG:    TEST8ri %10, 1, implicit-def $eflags
+; CHECK-VREG:    %19:gr64 = CMOV64rr %5, %4, 5, implicit $eflags
+; CHECK-VREG:    $rax = COPY %19
+; CHECK-VREG:    RET 0, $rax
+; CHECK-VREG:  bb.6.exceptional_return.left (landing-pad):
+; CHECK-VREG:    EH_LABEL <mcsymbol >
+; CHECK-VREG:    %18:gr64 = COPY killed $rdx
+; CHECK-VREG:    %17:gr64 = COPY killed $rax
+; CHECK-VREG:    $rax = COPY %16
+; CHECK-VREG:    RET 0, $rax
+; CHECK-VREG:  bb.7.exceptional_return.right (landing-pad):
+; CHECK-VREG:    EH_LABEL <mcsymbol >
+; CHECK-VREG:    %14:gr64 = COPY killed $rdx
+; CHECK-VREG:    %13:gr64 = COPY killed $rax
+; CHECK-VREG:    $rax = COPY %12
+; CHECK-VREG:    RET 0, $rax
+
+; CHECK-PREG-LABEL: name:            test_invoke_same_val
+; CHECK-PREG:  bb.0.entry:
+; CHECK-PREG:    renamable $r15 = COPY $rcx
+; CHECK-PREG:    renamable $rbx = COPY $rdx
+; CHECK-PREG:    renamable $rbp = COPY $rsi
+; CHECK-PREG:    renamable $r14d = COPY $edi
+; CHECK-PREG:    TEST8ri renamable $r14b, 1, implicit-def $eflags
+; CHECK-PREG:    JCC_1 %bb.3, 4, implicit killed $eflags
+; CHECK-PREG:    JMP_1 %bb.1
+; CHECK-PREG:  bb.1.left:
+; CHECK-PREG:    EH_LABEL <mcsymbol >
+; CHECK-PREG:    $rdi = COPY renamable $rbp
+; CHECK-PREG:    renamable $rbx, renamable $rbp = STATEPOINT 0, 0, 1, @some_call, $rdi, 2, 0, 2, 0, 2, 0, killed renamable $rbx, renamable $rbx(tied-def 0), killed renamable $rbp, renamable $rbp(tied-def 1), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-PREG:    EH_LABEL <mcsymbol >
+; CHECK-PREG:    JMP_1 %bb.2
+; CHECK-PREG:  bb.2.left.relocs:
+; CHECK-PREG:    JMP_1 %bb.5
+; CHECK-PREG:  bb.3.right:
+; CHECK-PREG:    EH_LABEL <mcsymbol >
+; CHECK-PREG:    $rdi = COPY killed renamable $rbp
+; CHECK-PREG:    renamable $r15, renamable $rbx = STATEPOINT 0, 0, 1, @some_call, $rdi, 2, 0, 2, 0, 2, 0, killed renamable $r15, renamable $r15(tied-def 0), killed renamable $rbx, renamable $rbx(tied-def 1), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-PREG:    EH_LABEL <mcsymbol >
+; CHECK-PREG:    JMP_1 %bb.4
+; CHECK-PREG:  bb.4.right.relocs:
+; CHECK-PREG:    renamable $rbp = COPY killed renamable $r15
+; CHECK-PREG:  bb.5.normal_return:
+; CHECK-PREG:    TEST8ri renamable $r14b, 1, implicit-def $eflags, implicit killed $r14d
+; CHECK-PREG:    renamable $rbp = CMOV64rr killed renamable $rbp, killed renamable $rbx, 4, implicit killed $eflags
+; CHECK-PREG:    $rax = COPY killed renamable $rbp
+; CHECK-PREG:    RET 0, $rax
+; CHECK-PREG:  bb.6.exceptional_return.left (landing-pad):
+; CHECK-PREG:    EH_LABEL <mcsymbol >
+; CHECK-PREG:    $rax = COPY killed renamable $rbp
+; CHECK-PREG:    RET 0, $rax
+; CHECK-PREG:  bb.7.exceptional_return.right (landing-pad):
+; CHECK-PREG:    EH_LABEL <mcsymbol >
+; CHECK-PREG:    $rax = COPY killed renamable $rbx
+; CHECK-PREG:    RET 0, $rax
+  gc "statepoint-example" personality i32 ()* @"personality_function" {
+entry:
+  br i1 %cond, label %left, label %right
+
+left:
+  %sp1 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 0) ["gc-live"(i64 addrspace(1)* %val1, i64 addrspace(1)* %val2)]
+           to label %left.relocs unwind label %exceptional_return.left
+
+left.relocs:
+  %val1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp1, i32 0, i32 0)
+  %val2.relocated_left = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp1, i32 1, i32 1)
+  br label %normal_return
+
+right:
+  %sp2 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 0) ["gc-live"(i64 addrspace(1)* %val2, i64 addrspace(1)* %val3)]
+           to label %right.relocs unwind label %exceptional_return.right
+
+right.relocs:
+  %val2.relocated_right = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp2, i32 0, i32 0)
+  %val3.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp2, i32 1, i32 1)
+  br label %normal_return
+
+normal_return:
+  %a1 = phi i64 addrspace(1)* [%val1.relocated, %left.relocs], [%val3.relocated, %right.relocs]
+  %a2 = phi i64 addrspace(1)* [%val2.relocated_left, %left.relocs], [%val2.relocated_right, %right.relocs]
+  %ret = select i1 %cond, i64 addrspace(1)* %a1, i64 addrspace(1)* %a2
+  ret i64 addrspace(1)* %ret
+
+exceptional_return.left:
+  %landing_pad = landingpad token
+          cleanup
+  %val.relocated2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 0, i32 0)
+  ret i64 addrspace(1)* %val.relocated2
+
+exceptional_return.right:
+  %landing_pad1 = landingpad token
+          cleanup
+  %val.relocated3 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad1, i32 0, i32 0)
+  ret i64 addrspace(1)* %val.relocated3
+}
+
 declare token @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
 declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...)
 declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32)
+declare i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token, i32, i32)
 declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32)
 declare <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token, i32, i32)
 declare i1 @llvm.experimental.gc.result.i1(token)
 
 ; CHECK-ASM-LABEL: .section .llvm_stackmaps
 ; CHECK-ASM-NEXT:  __LLVM_StackMaps:
 ; Entry for test_relocate
 ; CHECK-ASM:          .quad 0

 ; CHECK-ASM-NEXT: .long 0
 ; Location 3 Constant 0
 ; CHECK-ASM-NEXT: .byte 4
 ; CHECK-ASM-NEXT: .byte 0
 ; CHECK-ASM-NEXT: .short  8
 ; CHECK-ASM-NEXT: .short  0
 ; CHECK-ASM-NEXT: .short  0
 ; CHECK-ASM-NEXT: .long 0
-; Location 4 Indirect $rsp + 0
-; CHECK-ASM-NEXT: .byte 3
+; Location 4 Register $rbx
+; CHECK-ASM-NEXT: .byte 1
 ; CHECK-ASM-NEXT: .byte 0
 ; CHECK-ASM-NEXT: .short  8
-; CHECK-ASM-NEXT: .short  7
+; CHECK-ASM-NEXT: .short  3
 ; CHECK-ASM-NEXT: .short  0
 ; CHECK-ASM-NEXT: .long 0
-; Location 5 Indirect $rsp + 0
-; CHECK-ASM-NEXT: .byte 3
+; Location 5 Register $rbx
+; CHECK-ASM-NEXT: .byte 1
 ; CHECK-ASM-NEXT: .byte 0
 ; CHECK-ASM-NEXT: .short  8
-; CHECK-ASM-NEXT: .short  7
+; CHECK-ASM-NEXT: .short  3
 ; CHECK-ASM-NEXT: .short  0
 ; CHECK-ASM-NEXT: .long 0