Index: include/llvm/CodeGen/FunctionLoweringInfo.h =================================================================== --- include/llvm/CodeGen/FunctionLoweringInfo.h +++ include/llvm/CodeGen/FunctionLoweringInfo.h @@ -73,6 +73,14 @@ /// cross-basic-block values. DenseMap ValueMap; + // Keep track of frame indices allocated for invoke statepoint values + // used across basic block boundaries + // If first value of the pair is true then second will contain frame index. + // If first value of the pair is false it indicates that Value was visited + // but not spilled. It can happen for constant and allocas. We are using it + // for consistency checks. + DenseMap> InvokeStatepointValueSlots; + /// StaticAllocaMap - Keep track of frame indices for fixed sized allocas in /// the entry block. This allows the allocas to be efficiently referenced /// anywhere in the function. Index: lib/CodeGen/SelectionDAG/StatepointLowering.h =================================================================== --- lib/CodeGen/SelectionDAG/StatepointLowering.h +++ lib/CodeGen/SelectionDAG/StatepointLowering.h @@ -42,6 +42,11 @@ /// midst of processing a statepoint sequence. void clear(); + // This is separate from clear() method because sometimes we want to + // skip validation checks. This is what happens at the end of invoke statepoint + // lowering. + void clearValidationInfo(); + /// Returns the spill location of a value incoming to the current /// statepoint. Will return SDValue() if this value hasn't been /// spilled. Otherwise, the value has already been spilled and no Index: lib/CodeGen/SelectionDAG/StatepointLowering.cpp =================================================================== --- lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -54,12 +54,18 @@ AllocatedStackSlots[i] = false; } } + void StatepointLoweringState::clear() { + assert(PendingGCRelocateCalls.empty() && + "cleared before statepoint sequence completed"); Locations.clear(); RelocLocations.clear(); AllocatedStackSlots.clear(); - assert(PendingGCRelocateCalls.empty() && - "cleared before statepoint sequence completed"); + clearValidationInfo(); +} + +void StatepointLoweringState::clearValidationInfo() { + PendingGCRelocateCalls.clear(); } SDValue @@ -523,6 +529,30 @@ SDValue Incoming = Builder.getValue(V); lowerIncomingStatepointValue(Incoming, Ops, Builder); } + + // For invoke statepoint we need to export locations of spilled values. + if (StatepointSite.getCallSite().isInvoke()) { + for (GCRelocateOperands relocateOpers : + StatepointSite.getRelocates(StatepointSite)) { + const Value *V = relocateOpers.derivedPtr(); + SDValue sdV = Builder.getValue(V); + SDValue loc = Builder.StatepointLowering.getLocation(sdV); + + if (loc.getNode()) { + Builder.FuncInfo.InvokeStatepointValueSlots[V] = + std::make_pair(true, cast(loc)->getIndex()); + } + else { + Builder.FuncInfo.InvokeStatepointValueSlots[V] = + std::make_pair(false, 0); + + // Not spilled values should be exported. Otherwise we end up using + // CopyFromReg from undefined register in visitGcRelocate. + Builder.ExportFromCurrentBlock(V); + } + } + } + } void SelectionDAGBuilder::visitStatepoint(const CallInst &CI) { @@ -645,6 +675,12 @@ // Remove originall call node DAG.DeleteNode(CallNode); + // For invoke statepoint we don't use validation info stored + // in StatepointLowering. FuncInfo::InvokeStatepointValueSlots suits similar + // purposes for them. + if (CS.isInvoke()) + StatepointLowering.clearValidationInfo(); + // DON'T set the root - under the assumption that it's already set past the // inserted node we created. @@ -683,13 +719,21 @@ } void SelectionDAGBuilder::visitGCRelocate(const CallInst &CI) { + GCRelocateOperands relocateOpers(&CI); + #ifndef NDEBUG // Consistency check - StatepointLowering.relocCallVisited(CI); + // We skip this check for invoke statepoints. It would be too expensive to + // preserve validation info through different basic blocks. + // FuncInfo::InvokeStatepointValueSlots suits similar purposes for + // invoke statepoints. + if (!relocateOpers.isTiedToInvoke()) { + StatepointLowering.relocCallVisited(CI); + } #endif - GCRelocateOperands relocateOpers(&CI); - SDValue SD = getValue(relocateOpers.derivedPtr()); + const Value *DerivedPtr = relocateOpers.derivedPtr(); + SDValue SD = getValue(DerivedPtr); if (isa(SD) || isa(SD)) { // We didn't need to spill these special cases (constants and allocas). @@ -698,11 +742,34 @@ return; } + if ((relocateOpers.isTiedToInvoke() && + !FuncInfo.InvokeStatepointValueSlots[DerivedPtr].first)) { + // In this case we have visited invoke statepoint value + // but didn't spill it. This is somewhat similar to the previous case + // but we can not use isa<...> checks because SD will always be CopyFromReg + // node. + setValue(&CI, SD); + return; + } + SDValue Loc = StatepointLowering.getRelocLocation(SD); // Emit new load if we did not emit it before if (!Loc.getNode()) { - SDValue SpillSlot = StatepointLowering.getLocation(SD); - int FI = cast(SpillSlot)->getIndex(); + int FI = INT32_MAX; + SDValue SpillSlot; + + if (relocateOpers.isTiedToInvoke()) { + assert(FuncInfo.InvokeStatepointValueSlots.count(DerivedPtr)); + assert(FuncInfo.InvokeStatepointValueSlots[DerivedPtr].first); + + FI = FuncInfo.InvokeStatepointValueSlots[DerivedPtr].second; + SpillSlot = DAG.getTargetFrameIndex(FI, SD.getValueType()); + } + else { + SpillSlot = StatepointLowering.getLocation(SD); + FI = cast(SpillSlot)->getIndex(); + } + assert(SpillSlot.getNode() && FI != INT32_MAX); // Be conservative: flush all pending loads // TODO: Probably we can be less restrictive on this, Index: test/CodeGen/X86/statepoint-invoke.ll =================================================================== --- test/CodeGen/X86/statepoint-invoke.ll +++ test/CodeGen/X86/statepoint-invoke.ll @@ -1,38 +1,82 @@ ; RUN: llc < %s 2>&1 | FileCheck %s +declare void @"some_call"(i64 addrspace(1)*) declare i64 addrspace(1)* @"some_other_call"(i64 addrspace(1)*) declare i32 @"personality_function"() -define i64 addrspace(1)* @test_result(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) { +define i64 addrspace(1)* @test_basic(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) { entry: ; CHECK-LABEL: Ltmp0: - ; CHECK: callq _some_other_call + ; CHECK: callq _some_call ; CHECK-LABEL: Ltmp1: + %0 = invoke i32 (void (i64 addrspace(1)*)*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) + to label %invoke_safepoint_normal_dest unwind label %exceptional_return + +invoke_safepoint_normal_dest: + ; CHECK: movq + %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %0, i32 10, i32 10) + %obj1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %0, i32 11, i32 11) + br label %normal_return + +normal_return: + ; CHECK: retq + ret i64 addrspace(1)* %obj.relocated + +exceptional_return: + ; CHECK-LABEL: Ltmp2: + ; CHECK: movq + ; CHECK: retq + %landing_pad = landingpad { i8*, i32 } personality i32 ()* @"personality_function" + cleanup + %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1 + %obj.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 10, i32 10) + %obj1.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 11, i32 11) + ret i64 addrspace(1)* %obj1.relocated1 +} +; CHECK-LABEL: GCC_except_table0: +; CHECK: Lset1 = Ltmp1-Ltmp0 +; CHECK: .long Lset1 +; CHECK: Lset2 = Ltmp2-Leh_func_begin0 +; CHECK: .long Lset2 +; CHECK: .byte 0 +; CHECK: .align 2 + +define i64 addrspace(1)* @test_result(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) { +entry: + ; CHECK-LABEL: Ltmp5: + ; CHECK: callq _some_other_call + ; CHECK-LABEL: Ltmp6: %0 = invoke i32 (i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...)* @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 addrspace(1)* (i64 addrspace(1)*)* @some_other_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) to label %normal_return unwind label %exceptional_return normal_return: - ; CHECK: popq + ; CHECK: popq %rdx ; CHECK: retq %ret_val = call i64 addrspace(1)* @llvm.experimental.gc.result.p1i64(i32 %0) ret i64 addrspace(1)* %ret_val exceptional_return: - ; CHECK-LABEL: Ltmp2: + ; CHECK-LABEL: Ltmp7: + ; CHECK: movq ; CHECK: popq ; CHECK: retq %landing_pad = landingpad { i8*, i32 } personality i32 ()* @personality_function cleanup - ret i64 addrspace(1)* %obj + %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1 + %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 10, i32 10) + ret i64 addrspace(1)* %obj.relocated } -; CHECK-LABEL: GCC_except_table0: -; CHECK: Lset1 = Ltmp1-Ltmp0 -; CHECK: .long Lset1 -; CHECK: Lset2 = Ltmp2-Leh_func_begin0 -; CHECK: .long Lset2 +; CHECK-LABEL: GCC_except_table1: +; CHECK: Lset4 = Ltmp6-Ltmp5 +; CHECK: .long Lset4 +; CHECK: Lset5 = Ltmp7-Leh_func_begin1 +; CHECK: .long Lset5 ; CHECK: .byte 0 ; CHECK: .align 2 +declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(void (i64 addrspace(1)*)*, i32, i32, ...) declare i32 @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) + +declare i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32, i32, i32) declare i64 addrspace(1)* @llvm.experimental.gc.result.p1i64(i32)