Index: llvm/include/llvm/IR/IRBuilder.h =================================================================== --- llvm/include/llvm/IR/IRBuilder.h +++ llvm/include/llvm/IR/IRBuilder.h @@ -804,7 +804,7 @@ /// start a new statepoint sequence. CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags, - ArrayRef CallArgs, + ArrayRef CallArgs, Optional> TransitionArgs, Optional> DeoptArgs, ArrayRef GCArgs, @@ -833,7 +833,7 @@ InvokeInst *CreateGCStatepointInvoke( uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags, - ArrayRef InvokeArgs, Optional> TransitionArgs, + ArrayRef InvokeArgs, Optional> TransitionArgs, Optional> DeoptArgs, ArrayRef GCArgs, const Twine &Name = ""); Index: llvm/lib/IR/IRBuilder.cpp =================================================================== --- llvm/lib/IR/IRBuilder.cpp +++ llvm/lib/IR/IRBuilder.cpp @@ -663,10 +663,10 @@ CallInst *IRBuilderBase::CreateGCStatepointCall( uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags, - ArrayRef CallArgs, Optional> TransitionArgs, + ArrayRef CallArgs, Optional> TransitionArgs, Optional> DeoptArgs, ArrayRef GCArgs, const Twine &Name) { - return CreateGCStatepointCallCommon( + return CreateGCStatepointCallCommon( this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs, DeoptArgs, GCArgs, Name); } @@ -721,9 +721,9 @@ InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags, - ArrayRef InvokeArgs, Optional> TransitionArgs, + ArrayRef InvokeArgs, Optional> TransitionArgs, Optional> DeoptArgs, ArrayRef GCArgs, const Twine &Name) { - return CreateGCStatepointInvokeCommon( + return CreateGCStatepointInvokeCommon( this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags, InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name); } Index: llvm/lib/IR/Verifier.cpp =================================================================== --- llvm/lib/IR/Verifier.cpp +++ llvm/lib/IR/Verifier.cpp @@ -2284,11 +2284,15 @@ // Check function attributes. verifyFunctionAttrs(FT, Attrs, &F, isLLVMdotName); - // On function declarations/definitions, we do not support the builtin - // attribute. We do not check this in VerifyFunctionAttrs since that is - // checking for Attributes that can/can not ever be on functions. + // On function declarations/definitions, we do not support the builtin or + // requires-statepoint attribute. We do not check this in VerifyFunctionAttrs + // since that is checking for Attributes that can/can not ever be on + // functions. Assert(!Attrs.hasFnAttribute(Attribute::Builtin), "Attribute 'builtin' can only be applied to a callsite.", &F); + Assert(!Attrs.hasFnAttribute("requires-statepoint"), + "Attribute 'requires-statepoint' can only be applied to a callsite.", + &F); // Check that this function meets the restrictions on this calling convention. // Sometimes varargs is used for perfectly forwarding thunks, so some of these @@ -3011,6 +3015,15 @@ "llvm.call.preallocated.arg"); } + if (Attrs.hasFnAttribute("requires-statepoint")) { + auto IID = Call.getCalledFunction()->getIntrinsicID(); + Assert(IID == Intrinsic::memcpy_element_unordered_atomic || + IID == Intrinsic::memmove_element_unordered_atomic, + "requires-statepoint attribute can only be on " + "llvm.memcpy.element.unordered.atomic or " + "llvm.memmove.element.unordered.atomic calls"); + } + // Verify call attributes. verifyFunctionAttrs(FTy, Attrs, &Call, IsIntrinsic); Index: llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp =================================================================== --- llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp +++ llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp @@ -1306,6 +1306,7 @@ AttrBuilder FnAttrs = AL.getFnAttributes(); FnAttrs.removeAttribute(Attribute::ReadNone); FnAttrs.removeAttribute(Attribute::ReadOnly); + FnAttrs.removeAttribute("requires-statepoint"); for (Attribute A : AL.getFnAttributes()) { if (isStatepointDirectiveAttr(A)) FnAttrs.remove(A); @@ -1487,7 +1488,9 @@ uint32_t NumPatchBytes = 0; uint32_t Flags = uint32_t(StatepointFlags::None); - ArrayRef CallArgs(Call->arg_begin(), Call->arg_end()); + SmallVector CallArgs; + for (Value *Arg : Call->args()) + CallArgs.push_back(Arg); Optional> DeoptArgs; if (auto Bundle = Call->getOperandBundle(LLVMContext::OB_deopt)) DeoptArgs = Bundle->Inputs; @@ -1520,7 +1523,8 @@ Value *CallTarget = Call->getCalledOperand(); if (Function *F = dyn_cast(CallTarget)) { - if (F->getIntrinsicID() == Intrinsic::experimental_deoptimize) { + auto IID = F->getIntrinsicID(); + if (IID == Intrinsic::experimental_deoptimize) { // Calls to llvm.experimental.deoptimize are lowered to calls to the // __llvm_deoptimize symbol. We want to resolve this now, since the // verifier does not allow taking the address of an intrinsic function. @@ -1540,6 +1544,85 @@ .getCallee(); IsDeoptimize = true; + } else if (IID == Intrinsic::memcpy_element_unordered_atomic || + IID == Intrinsic::memmove_element_unordered_atomic) { + assert(Call->hasFnAttr("requires-statepoint") && + "Should be a GC leaf call otherwise!"); + // We are explicitly asked to produce a GC parseable memcpy or memmove. + // These calls should be lowered to the + // __llvm_{memcpy|memmove}_element_unordered_atomic_safepoint symbols. + // Similarly to __llvm_deoptimize we want to resolve this now, since the + // verifier does not allow taking the address of an intrinsic function. + // Moreover we need to shuffle the arguments for the call to make it GC + // parseable. + // + // In addition to derived pointers for src and dest we also need to pass + // the corresponding base pointers. This way the implementation of the + // intrinsic knows how to relocate the pointers should the GC occur half + // way through the operation. + auto *Dest = CallArgs[0]; + assert(Result.PointerToBase.count(Dest)); + Value *DestBase = Result.PointerToBase.find(Dest)->second; + + auto *Source = CallArgs[1]; + assert(Result.PointerToBase.count(Source)); + Value *SourceBase = Result.PointerToBase.find(Source)->second; + + auto *LengthInBytes = CallArgs[2]; + auto *ElementSizeCI = cast(CallArgs[3]); + + CallArgs.clear(); + CallArgs.push_back(DestBase); + CallArgs.push_back(Dest); + CallArgs.push_back(SourceBase); + CallArgs.push_back(Source); + CallArgs.push_back(LengthInBytes); + + SmallVector DomainTy; + for (Value *Arg : CallArgs) + DomainTy.push_back(Arg->getType()); + auto *FTy = FunctionType::get(Type::getVoidTy(F->getContext()), DomainTy, + /* isVarArg = */ false); + + auto GetFunctionName = [](Intrinsic::ID IID, ConstantInt *ElementSizeCI) { + uint64_t ElementSize = ElementSizeCI->getZExtValue(); + if (IID == Intrinsic::memcpy_element_unordered_atomic) { + switch (ElementSize) { + case 1: + return "__llvm_memcpy_element_unordered_atomic_safepoint_1"; + case 2: + return "__llvm_memcpy_element_unordered_atomic_safepoint_2"; + case 4: + return "__llvm_memcpy_element_unordered_atomic_safepoint_4"; + case 8: + return "__llvm_memcpy_element_unordered_atomic_safepoint_8"; + case 16: + return "__llvm_memcpy_element_unordered_atomic_safepoint_16"; + default: + llvm_unreachable("unexpected element size!"); + } + } + assert(IID == Intrinsic::memmove_element_unordered_atomic); + switch (ElementSize) { + case 1: + return "__llvm_memmove_element_unordered_atomic_safepoint_1"; + case 2: + return "__llvm_memmove_element_unordered_atomic_safepoint_2"; + case 4: + return "__llvm_memmove_element_unordered_atomic_safepoint_4"; + case 8: + return "__llvm_memmove_element_unordered_atomic_safepoint_8"; + case 16: + return "__llvm_memmove_element_unordered_atomic_safepoint_16"; + default: + llvm_unreachable("unexpected element size!"); + } + }; + + CallTarget = + F->getParent() + ->getOrInsertFunction(GetFunctionName(IID, ElementSizeCI), FTy) + .getCallee(); } } Index: llvm/lib/Transforms/Utils/Local.cpp =================================================================== --- llvm/lib/Transforms/Utils/Local.cpp +++ llvm/lib/Transforms/Utils/Local.cpp @@ -2673,10 +2673,16 @@ if (F->hasFnAttribute("gc-leaf-function")) return true; - if (auto IID = F->getIntrinsicID()) + if (auto IID = F->getIntrinsicID()) { + // These intrinsics can be marked explicitly to require statepoint. + if (IID == Intrinsic::memcpy_element_unordered_atomic || + IID == Intrinsic::memmove_element_unordered_atomic) + return !Call->hasFnAttr("requires-statepoint"); + // Most LLVM intrinsics do not take safepoints. return IID != Intrinsic::experimental_gc_statepoint && IID != Intrinsic::experimental_deoptimize; + } } // Lib calls can be materialized by some passes, and won't be