Index: include/llvm/CodeGen/SelectionDAG.h =================================================================== --- include/llvm/CodeGen/SelectionDAG.h +++ include/llvm/CodeGen/SelectionDAG.h @@ -1150,7 +1150,8 @@ /// Create a stack temporary, suitable for holding the /// specified value type. If minAlign is specified, the slot size will have /// at least that alignment. - SDValue CreateStackTemporary(EVT VT, unsigned minAlign = 1); + SDValue CreateStackTemporary(EVT VT, unsigned minAlign = 1, + bool isSS = false); /// Create a stack temporary suitable for holding /// either of the specified value types. Index: include/llvm/IR/Intrinsics.td =================================================================== --- include/llvm/IR/Intrinsics.td +++ include/llvm/IR/Intrinsics.td @@ -575,7 +575,7 @@ def int_experimental_gc_result : Intrinsic<[llvm_any_ty], [llvm_i32_ty], [IntrReadMem]>; -def int_experimental_gc_relocate : Intrinsic<[llvm_anyptr_ty], +def int_experimental_gc_relocate : Intrinsic<[llvm_any_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrReadMem]>; Index: lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -1888,14 +1888,15 @@ /// CreateStackTemporary - Create a stack temporary, suitable for holding the /// specified value type. -SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { +SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign, + bool isSS) { MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); unsigned ByteSize = VT.getStoreSize(); Type *Ty = VT.getTypeForEVT(*getContext()); unsigned StackAlign = std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign); - int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false); + int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, isSS); return getFrameIndex(FrameIdx, TLI->getPointerTy(getDataLayout())); } Index: lib/CodeGen/SelectionDAG/StatepointLowering.cpp =================================================================== --- lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -93,7 +93,7 @@ StatepointMaxSlotsRequired = NumSlots + 1; } - SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType); + SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType, 1, true); const unsigned FI = cast(SpillSlot)->getIndex(); Builder.FuncInfo.StatepointStackSlots.push_back(FI); AllocatedStackSlots.push_back(true); @@ -507,8 +507,13 @@ // heap. This is basically just here to help catch errors during statepoint // insertion. TODO: This should actually be in the Verifier, but we can't get // to the GCStrategy from there (yet). + // TODO: This needs to become a predicate on the Type so we can call + // getScalarType here. The current hack around for vectors needs fixed + // before submission. GCStrategy &S = Builder.GFI->getStrategy(); for (const Value *V : Bases) { + if (V->getType()->isVectorTy()) + continue; auto Opt = S.isGCManagedPointer(V); if (Opt.hasValue()) { assert(Opt.getValue() && @@ -516,6 +521,8 @@ } } for (const Value *V : Ptrs) { + if (V->getType()->isVectorTy()) + continue; auto Opt = S.isGCManagedPointer(V); if (Opt.hasValue()) { assert(Opt.getValue() && @@ -523,6 +530,8 @@ } } for (const Value *V : Relocations) { + if (V->getType()->isVectorTy()) + continue; auto Opt = S.isGCManagedPointer(V); if (Opt.hasValue()) { assert(Opt.getValue() && "non gc managed pointer relocated"); Index: lib/CodeGen/TargetLoweringBase.cpp =================================================================== --- lib/CodeGen/TargetLoweringBase.cpp +++ lib/CodeGen/TargetLoweringBase.cpp @@ -1094,6 +1094,7 @@ TargetLoweringBase::emitPatchPoint(MachineInstr *MI, MachineBasicBlock *MBB) const { MachineFunction &MF = *MI->getParent()->getParent(); + const MachineFrameInfo &MFI = *MF.getFrameInfo(); // MI changes inside this loop as we grow operands. for(unsigned OperIdx = 0; OperIdx != MI->getNumOperands(); ++OperIdx) { @@ -1109,10 +1110,19 @@ // Copy operands before the frame-index. for (unsigned i = 0; i < OperIdx; ++i) MIB.addOperand(MI->getOperand(i)); - // Add frame index operands: direct-mem-ref tag, #FI, offset. - MIB.addImm(StackMaps::DirectMemRefOp); - MIB.addOperand(MI->getOperand(OperIdx)); - MIB.addImm(0); + // Add frame index operands recognized by stackmaps.cpp + if (MFI.isSpillSlotObjectIndex(FI)) { + // indirect-mem-ref tag, size, #FI, offset. + MIB.addImm(StackMaps::IndirectMemRefOp); + MIB.addImm(MFI.getObjectSize(FI)); + MIB.addOperand(MI->getOperand(OperIdx)); + MIB.addImm(0); + } else { + // direct-mem-ref tag, #FI, offset. + MIB.addImm(StackMaps::DirectMemRefOp); + MIB.addOperand(MI->getOperand(OperIdx)); + MIB.addImm(0); + } // Copy the operands after the frame index. for (unsigned i = OperIdx + 1; i != MI->getNumOperands(); ++i) MIB.addOperand(MI->getOperand(i)); @@ -1122,7 +1132,6 @@ assert(MIB->mayLoad() && "Folded a stackmap use to a non-load!"); // Add a new memory operand for this FI. - const MachineFrameInfo &MFI = *MF.getFrameInfo(); assert(MFI.getObjectOffset(FI) != -1); unsigned Flags = MachineMemOperand::MOLoad; Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -3647,6 +3647,9 @@ case Intrinsic::experimental_gc_relocate: { Assert(CS.getNumArgOperands() == 3, "wrong number of arguments", CS); + Assert(isa(CS.getType()->getScalarType()), + "gc.relocate must return a pointer or a vector of pointers", CS); + // Check that this relocate is correctly tied to the statepoint // This is case for relocate on the unwinding path of an invoke statepoint @@ -3732,17 +3735,20 @@ "'gc parameters' section of the statepoint call", CS); - // Relocated value must be a pointer type, but gc_relocate does not need to return the - // same pointer type as the relocated pointer. It can be casted to the correct type later - // if it's desired. However, they must have the same address space. + // Relocated value must be either a pointer type or vector-of-pointer type, + // but gc_relocate does not need to return the same pointer type as the + // relocated pointer. It can be casted to the correct type later if it's + // desired. However, they must have the same address space and 'vectorness'. GCRelocateOperands Operands(CS); - Assert(Operands.getDerivedPtr()->getType()->isPointerTy(), + Assert(Operands.getDerivedPtr()->getType()->getScalarType()->isPointerTy(), "gc.relocate: relocated value must be a gc pointer", CS); - // gc_relocate return type must be a pointer type, and is verified earlier in - // VerifyIntrinsicType(). - Assert(cast(CS.getType())->getAddressSpace() == - cast(Operands.getDerivedPtr()->getType())->getAddressSpace(), + auto ResultType = CS.getType(); + auto DerivedType = Operands.getDerivedPtr()->getType(); + Assert(ResultType->isVectorTy() == DerivedType->isVectorTy(), + "gc.relocate: vector relocates to vector and pointer to pointer", CS); + Assert(ResultType->getPointerAddressSpace() == + DerivedType->getPointerAddressSpace(), "gc.relocate: relocating a pointer shouldn't change its address space", CS); break; }