Index: include/llvm/CodeGen/FunctionLoweringInfo.h =================================================================== --- include/llvm/CodeGen/FunctionLoweringInfo.h +++ include/llvm/CodeGen/FunctionLoweringInfo.h @@ -73,6 +73,14 @@ /// cross-basic-block values. DenseMap ValueMap; + // Keep track of frame indices allocated for invoke statepoint values + // used across basic block boundaries + // If first value of the pair is true then second will contain frame index. + // If first value of the pair is false it indicates that Value was visited + // but not spilled. It can happen for constant and allocas. We are using it + // for consistency checks. + DenseMap> InvokeStatepointValueSlots; + /// StaticAllocaMap - Keep track of frame indices for fixed sized allocas in /// the entry block. This allows the allocas to be efficiently referenced /// anywhere in the function. Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -2039,13 +2039,21 @@ case Intrinsic::experimental_patchpoint_i64: visitPatchpoint(&I, LandingPad); break; + case Intrinsic::experimental_gc_statepoint: + LowerStatepoint(ImmutableStatepoint(&I), LandingPad); + break; } } else LowerCallTo(&I, getValue(Callee), false, LandingPad); // If the value of the invoke is used outside of its defining block, make it // available as a virtual register. - CopyToExportRegsIfNeeded(&I); + // For statepoints exported value exists only during LowerStatepoint so we + // can not export it here. Instead it was copied into preallocated register + // during LowerStatepoint (lowerCallFromStatepoint to be precise). + if (!isStatepoint(I)) { + CopyToExportRegsIfNeeded(&I); + } // Update successor info addSuccessorWithWeight(InvokeMBB, Return); Index: lib/CodeGen/SelectionDAG/StatepointLowering.h =================================================================== --- lib/CodeGen/SelectionDAG/StatepointLowering.h +++ lib/CodeGen/SelectionDAG/StatepointLowering.h @@ -42,6 +42,11 @@ /// midst of processing a statepoint sequence. void clear(); + // This is separate from clear() method because sometimes we want to + // skip validation checks. This is what happens at the end of invoke statepoint + // lowering. + void clearValidationInfo(); + /// Returns the spill location of a value incoming to the current /// statepoint. Will return SDValue() if this value hasn't been /// spilled. Otherwise, the value has already been spilled and no Index: lib/CodeGen/SelectionDAG/StatepointLowering.cpp =================================================================== --- lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -54,12 +54,18 @@ AllocatedStackSlots[i] = false; } } + void StatepointLoweringState::clear() { + assert(PendingGCRelocateCalls.empty() && + "cleared before statepoint sequence completed"); Locations.clear(); RelocLocations.clear(); AllocatedStackSlots.clear(); - assert(PendingGCRelocateCalls.empty() && - "cleared before statepoint sequence completed"); + clearValidationInfo(); +} + +void StatepointLoweringState::clearValidationInfo() { + PendingGCRelocateCalls.clear(); } SDValue @@ -524,6 +530,30 @@ SDValue Incoming = Builder.getValue(V); lowerIncomingStatepointValue(Incoming, Ops, Builder); } + + // For invoke statepoint we need to export locations of spilled values. + if (StatepointSite.getCallSite().isInvoke()) { + for (GCRelocateOperands relocateOpers : + StatepointSite.getRelocates(StatepointSite)) { + const Value *V = relocateOpers.derivedPtr(); + SDValue sdV = Builder.getValue(V); + SDValue loc = Builder.StatepointLowering.getLocation(sdV); + + if (loc.getNode()) { + Builder.FuncInfo.InvokeStatepointValueSlots[V] = + std::make_pair(true, cast(loc)->getIndex()); + } + else { + Builder.FuncInfo.InvokeStatepointValueSlots[V] = + std::make_pair(false, 0); + + // Not spilled values should be exported. Otherwise we end up using + // CopyFromReg from undefined register in visitGcRelocate. + Builder.ExportFromCurrentBlock(V); + } + } + } + } void SelectionDAGBuilder::visitStatepoint(const CallInst &CI) { @@ -646,6 +676,12 @@ // Remove originall call node DAG.DeleteNode(CallNode); + // For invoke statepoint we don't use validation info stored + // in StatepointLowering. FuncInfo::InvokeStatepointValueSlots suits similar + // purposes for them. + if (CS.isInvoke()) + StatepointLowering.clearValidationInfo(); + // DON'T set the root - under the assumption that it's already set past the // inserted node we created. @@ -684,13 +720,21 @@ } void SelectionDAGBuilder::visitGCRelocate(const CallInst &CI) { + GCRelocateOperands relocateOpers(&CI); + #ifndef NDEBUG // Consistency check - StatepointLowering.relocCallVisited(CI); + // We skip this check for invoke statepoints. It would be too expensive to + // preserve validation info through different basic blocks. + // FuncInfo::InvokeStatepointValueSlots suits similar purposes for + // invoke statepoints. + if (!relocateOpers.isTiedToInvoke()) { + StatepointLowering.relocCallVisited(CI); + } #endif - GCRelocateOperands relocateOpers(&CI); - SDValue SD = getValue(relocateOpers.derivedPtr()); + const Value *DerivedPtr = relocateOpers.derivedPtr(); + SDValue SD = getValue(DerivedPtr); if (isa(SD) || isa(SD)) { // We didn't need to spill these special cases (constants and allocas). @@ -699,11 +743,34 @@ return; } + if ((relocateOpers.isTiedToInvoke() && + !FuncInfo.InvokeStatepointValueSlots[DerivedPtr].first)) { + // In this case we have visited invoke statepoint value + // but didn't spill it. This is somewhat similar to the previous case + // but we can not use isa<...> checks because SD will always be CopyFromReg + // node. + setValue(&CI, SD); + return; + } + SDValue Loc = StatepointLowering.getRelocLocation(SD); // Emit new load if we did not emit it before if (!Loc.getNode()) { - SDValue SpillSlot = StatepointLowering.getLocation(SD); - int FI = cast(SpillSlot)->getIndex(); + int FI = INT32_MAX; + SDValue SpillSlot; + + if (relocateOpers.isTiedToInvoke()) { + assert(FuncInfo.InvokeStatepointValueSlots.count(DerivedPtr)); + assert(FuncInfo.InvokeStatepointValueSlots[DerivedPtr].first); + + FI = FuncInfo.InvokeStatepointValueSlots[DerivedPtr].second; + SpillSlot = DAG.getTargetFrameIndex(FI, SD.getValueType()); + } + else { + SpillSlot = StatepointLowering.getLocation(SD); + FI = cast(SpillSlot)->getIndex(); + } + assert(SpillSlot.getNode() && FI != INT32_MAX); // Be conservative: flush all pending loads // TODO: Probably we can be less restrictive on this, Index: test/CodeGen/X86/statepoint-invoke.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/statepoint-invoke.ll @@ -0,0 +1,82 @@ +; RUN: llc < %s 2>&1 | FileCheck %s + +declare void @"some_call"(i64 addrspace(1)*) +declare i64 addrspace(1)* @"some_other_call"(i64 addrspace(1)*) + +declare i32 @"personality_function"() + +define i64 addrspace(1)* @test_basic(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) { +entry: + ; CHECK-LABEL: Ltmp0: + ; CHECK: callq _some_call + ; CHECK-LABEL: Ltmp1: + %0 = invoke i32 (void (i64 addrspace(1)*)*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) + to label %invoke_safepoint_normal_dest unwind label %exceptional_return + +invoke_safepoint_normal_dest: + ; CHECK: movq + %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %0, i32 10, i32 10) + %obj1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %0, i32 11, i32 11) + br label %normal_return + +normal_return: + ; CHECK: retq + ret i64 addrspace(1)* %obj.relocated + +exceptional_return: + ; CHECK-LABEL: Ltmp2: + ; CHECK: movq + ; CHECK: retq + %landing_pad = landingpad { i8*, i32 } personality i32 ()* @"personality_function" + cleanup + %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1 + %obj.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 10, i32 10) + %obj1.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 11, i32 11) + ret i64 addrspace(1)* %obj1.relocated1 +} +; CHECK-LABEL: GCC_except_table0: +; CHECK: Lset1 = Ltmp1-Ltmp0 +; CHECK: .long Lset1 +; CHECK: Lset2 = Ltmp2-Leh_func_begin0 +; CHECK: .long Lset2 +; CHECK: .byte 0 +; CHECK: .align 2 + +define i64 addrspace(1)* @test_result(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) { +entry: + ; CHECK-LABEL: Ltmp5: + ; CHECK: callq _some_other_call + ; CHECK-LABEL: Ltmp6: + %0 = invoke i32 (i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 addrspace(1)* (i64 addrspace(1)*)* @some_other_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) + to label %normal_return unwind label %exceptional_return + +normal_return: + ; CHECK: popq %rdx + ; CHECK: retq + %ret_val = call i64 addrspace(1)* @llvm.experimental.gc.result.p1i64(i32 %0) + ret i64 addrspace(1)* %ret_val + +exceptional_return: + ; CHECK-LABEL: Ltmp7: + ; CHECK: movq + ; CHECK: popq + ; CHECK: retq + %landing_pad = landingpad { i8*, i32 } personality i32 ()* @personality_function + cleanup + %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1 + %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 10, i32 10) + ret i64 addrspace(1)* %obj.relocated +} +; CHECK-LABEL: GCC_except_table1: +; CHECK: Lset4 = Ltmp6-Ltmp5 +; CHECK: .long Lset4 +; CHECK: Lset5 = Ltmp7-Leh_func_begin1 +; CHECK: .long Lset5 +; CHECK: .byte 0 +; CHECK: .align 2 + +declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(void (i64 addrspace(1)*)*, i32, i32, ...) +declare i32 @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) + +declare i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32, i32, i32) +declare i64 addrspace(1)* @llvm.experimental.gc.result.p1i64(i32)