Index: llvm/trunk/docs/LangRef.rst =================================================================== --- llvm/trunk/docs/LangRef.rst +++ llvm/trunk/docs/LangRef.rst @@ -7281,6 +7281,56 @@ other aggressive transformations, so the value returned may not be that of the obvious source-language caller. +'``llvm.frameallocate``' and '``llvm.recoverframeallocation``' Intrinsics +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare i8* @llvm.frameallocate(i32 %size) + declare i8* @llvm.recoverframeallocation(i8* %func, i8* %fp) + +Overview: +""""""""" + +The '``llvm.frameallocate``' intrinsic allocates stack memory at some fixed +offset from the frame pointer, and the '``llvm.recoverframeallocation``' +intrinsic applies that offset to a live frame pointer to recover the address of +the allocation. The offset is computed during frame layout of the caller of +``llvm.frameallocate``. + +Arguments: +"""""""""" + +The ``size`` argument to '``llvm.frameallocate``' must be a constant integer +indicating the amount of stack memory to allocate. As with allocas, allocating +zero bytes is legal, but the result is undefined. + +The ``func`` argument to '``llvm.recoverframeallocation``' must be a constant +bitcasted pointer to a function defined in the current module. The code +generator cannot determine the frame allocation offset of functions defined in +other modules. + +The ``fp`` argument to '``llvm.recoverframeallocation``' must be a frame +pointer of a call frame that is currently live. The return value of +'``llvm.frameaddress``' is one way to produce such a value, but most platforms +also expose the frame pointer through stack unwinding mechanisms. + +Semantics: +"""""""""" + +These intrinsics allow a group of functions to access one stack memory +allocation in an ancestor stack frame. The memory returned from +'``llvm.frameallocate``' may be allocated prior to stack realignment, so the +memory is only aligned to the ABI-required stack alignment. Each function may +only call '``llvm.frameallocate``' one or zero times from the function entry +block. The frame allocation intrinsic inhibits inlining, as any frame +allocations in the inlined function frame are likely to be at a different +offset from the one used by '``llvm.recoverframeallocation``' called with the +uninlined function. + .. _int_read_register: .. _int_write_register: Index: llvm/trunk/include/llvm/CodeGen/AsmPrinter.h =================================================================== --- llvm/trunk/include/llvm/CodeGen/AsmPrinter.h +++ llvm/trunk/include/llvm/CodeGen/AsmPrinter.h @@ -205,6 +205,8 @@ void emitCFIInstruction(const MachineInstr &MI); + void emitFrameAlloc(const MachineInstr &MI); + enum CFIMoveType { CFI_M_None, CFI_M_EH, CFI_M_Debug }; CFIMoveType needsCFIMoves(); Index: llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h =================================================================== --- llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h +++ llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h @@ -72,6 +72,11 @@ /// the parent's frame or return address, and so on. FRAMEADDR, RETURNADDR, + /// RECOVER_FRAME_ALLOC - Represents the llvm.recoverframeallocation + /// intrinsic. Materializes the offset from the frame pointer of another + /// function to the result of llvm.frameallocate. + RECOVER_FRAME_ALLOC, + /// READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on /// the DAG, which implements the named register global variables extension. READ_REGISTER, Index: llvm/trunk/include/llvm/CodeGen/MachineFrameInfo.h =================================================================== --- llvm/trunk/include/llvm/CodeGen/MachineFrameInfo.h +++ llvm/trunk/include/llvm/CodeGen/MachineFrameInfo.h @@ -516,6 +516,10 @@ /// on the stack. Returns an index with a negative value. int CreateFixedSpillStackObject(uint64_t Size, int64_t SPOffset); + /// Allocates memory at a fixed, target-specific offset from the frame + /// pointer. Marks the function as having its frame address taken. + int CreateFrameAllocation(uint64_t Size); + /// isFixedObjectIndex - Returns true if the specified index corresponds to a /// fixed stack object. bool isFixedObjectIndex(int ObjectIdx) const { Index: llvm/trunk/include/llvm/IR/Intrinsics.td =================================================================== --- llvm/trunk/include/llvm/IR/Intrinsics.td +++ llvm/trunk/include/llvm/IR/Intrinsics.td @@ -259,6 +259,10 @@ // def int_returnaddress : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty], [IntrNoMem]>; def int_frameaddress : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_frameallocate : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty]>; +def int_recoverframeallocation : Intrinsic<[llvm_ptr_ty], + [llvm_ptr_ty, llvm_ptr_ty], + [IntrNoMem]>; def int_read_register : Intrinsic<[llvm_anyint_ty], [llvm_metadata_ty], [IntrNoMem], "llvm.read_register">; def int_write_register : Intrinsic<[], [llvm_metadata_ty, llvm_anyint_ty], Index: llvm/trunk/include/llvm/MC/MCContext.h =================================================================== --- llvm/trunk/include/llvm/MC/MCContext.h +++ llvm/trunk/include/llvm/MC/MCContext.h @@ -237,6 +237,8 @@ MCSymbol *getOrCreateSectionSymbol(const MCSectionELF &Section); + MCSymbol *getOrCreateFrameAllocSymbol(StringRef FuncName); + /// LookupSymbol - Get the symbol for \p Name, or null. MCSymbol *LookupSymbol(StringRef Name) const; MCSymbol *LookupSymbol(const Twine &Name) const; Index: llvm/trunk/include/llvm/Target/Target.td =================================================================== --- llvm/trunk/include/llvm/Target/Target.td +++ llvm/trunk/include/llvm/Target/Target.td @@ -865,6 +865,15 @@ let hasSideEffects = 0; bit isPseudo = 1; } +def FRAME_ALLOC : Instruction { + // This instruction is really just a label. It has to be part of the chain so + // that it doesn't get dropped from the DAG, but it produces nothing and has + // no side effects. + let OutOperandList = (outs); + let InOperandList = (ins ptr_rc:$symbol, i32imm:$id); + let hasSideEffects = 0; + let hasCtrlDep = 1; +} } //===----------------------------------------------------------------------===// Index: llvm/trunk/include/llvm/Target/TargetFrameLowering.h =================================================================== --- llvm/trunk/include/llvm/Target/TargetFrameLowering.h +++ llvm/trunk/include/llvm/Target/TargetFrameLowering.h @@ -142,6 +142,10 @@ /// the assembly prologue to explicitly handle the stack. virtual void adjustForHiPEPrologue(MachineFunction &MF) const { } + /// Adjust the prologue to add an allocation at a fixed offset from the frame + /// pointer. + virtual void adjustForFrameAllocatePrologue(MachineFunction &MF) const { } + /// spillCalleeSavedRegisters - Issues instruction(s) to spill all callee /// saved registers and returns true if it isn't possible / profitable to do /// so by issuing a series of store instructions via Index: llvm/trunk/include/llvm/Target/TargetOpcodes.h =================================================================== --- llvm/trunk/include/llvm/Target/TargetOpcodes.h +++ llvm/trunk/include/llvm/Target/TargetOpcodes.h @@ -116,7 +116,12 @@ /// of live pointers for relocation by the garbage collector. It is /// intended to support garbage collection with fully precise relocating /// collectors and deoptimizations in either the callee or caller. - STATEPOINT = 20 + STATEPOINT = 20, + + /// Instruction that records the offset of a function's frame allocation in a + /// label. Created by the llvm.frameallocate intrinsic. It has two arguments: + /// the symbol for the label and the frame index of the stack allocation. + FRAME_ALLOC = 21, }; } // end namespace TargetOpcode } // end namespace llvm Index: llvm/trunk/lib/CodeGen/AsmPrinter/AsmPrinter.cpp =================================================================== --- llvm/trunk/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ llvm/trunk/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -748,6 +748,16 @@ emitCFIInstruction(CFI); } +void AsmPrinter::emitFrameAlloc(const MachineInstr &MI) { + // The operands are the MCSymbol and the frame offset of the allocation. + MCSymbol *FrameAllocSym = MI.getOperand(0).getMCSymbol(); + int FrameOffset = MI.getOperand(1).getImm(); + + // Emit a symbol assignment. + OutStreamer.EmitAssignment(FrameAllocSym, + MCConstantExpr::Create(FrameOffset, OutContext)); +} + /// EmitFunctionBody - This method emits the body and trailer for a /// function. void AsmPrinter::EmitFunctionBody() { @@ -786,6 +796,10 @@ emitCFIInstruction(MI); break; + case TargetOpcode::FRAME_ALLOC: + emitFrameAlloc(MI); + break; + case TargetOpcode::EH_LABEL: case TargetOpcode::GC_LABEL: OutStreamer.EmitLabel(MI.getOperand(0).getMCSymbol()); Index: llvm/trunk/lib/CodeGen/DeadMachineInstructionElim.cpp =================================================================== --- llvm/trunk/lib/CodeGen/DeadMachineInstructionElim.cpp +++ llvm/trunk/lib/CodeGen/DeadMachineInstructionElim.cpp @@ -59,6 +59,10 @@ if (MI->isInlineAsm()) return false; + // Don't delete frame allocation labels. + if (MI->getOpcode() == TargetOpcode::FRAME_ALLOC) + return false; + // Don't delete instructions with side effects. bool SawStore = false; if (!MI->isSafeToMove(TII, nullptr, SawStore) && !MI->isPHI()) Index: llvm/trunk/lib/CodeGen/MachineFunction.cpp =================================================================== --- llvm/trunk/lib/CodeGen/MachineFunction.cpp +++ llvm/trunk/lib/CodeGen/MachineFunction.cpp @@ -587,6 +587,14 @@ return -++NumFixedObjects; } +int MachineFrameInfo::CreateFrameAllocation(uint64_t Size) { + // Force the use of a frame pointer. The intention is that this intrinsic be + // used in conjunction with unwind mechanisms that leak the frame pointer. + setFrameAddressIsTaken(true); + Size = RoundUpToAlignment(Size, StackAlignment); + return CreateStackObject(Size, StackAlignment, false); +} + BitVector MachineFrameInfo::getPristineRegs(const MachineBasicBlock *MBB) const { assert(MBB && "MBB must be valid"); Index: llvm/trunk/lib/CodeGen/PrologEpilogInserter.cpp =================================================================== --- llvm/trunk/lib/CodeGen/PrologEpilogInserter.cpp +++ llvm/trunk/lib/CodeGen/PrologEpilogInserter.cpp @@ -817,6 +817,17 @@ continue; } + // Frame allocations are target independent. Simply swap the index with + // the offset. + if (MI->getOpcode() == TargetOpcode::FRAME_ALLOC) { + assert(TFI->hasFP(Fn) && "frame alloc requires FP"); + MachineOperand &FI = MI->getOperand(i); + unsigned Reg; + int FrameOffset = TFI->getFrameIndexReference(Fn, FI.getIndex(), Reg); + FI.ChangeToImmediate(FrameOffset); + continue; + } + // Some instructions (e.g. inline asm instructions) can have // multiple frame indices and/or cause eliminateFrameIndex // to insert more than one instruction. We need the register Index: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -48,6 +48,7 @@ #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Statepoint.h" +#include "llvm/MC/MCSymbol.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" @@ -5581,6 +5582,58 @@ } case Intrinsic::instrprof_increment: llvm_unreachable("instrprof failed to lower an increment"); + + case Intrinsic::frameallocate: { + MachineFunction &MF = DAG.getMachineFunction(); + const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); + + // Do the allocation and map it as a normal value. + // FIXME: Maybe we should add this to the alloca map so that we don't have + // to register allocate it? + uint64_t Size = cast(I.getArgOperand(0))->getZExtValue(); + int Alloc = MF.getFrameInfo()->CreateFrameAllocation(Size); + MVT PtrVT = TLI.getPointerTy(0); + SDValue FIVal = DAG.getFrameIndex(Alloc, PtrVT); + setValue(&I, FIVal); + + // Directly emit a FRAME_ALLOC machine instr. Label assignment emission is + // the same on all targets. + MCSymbol *FrameAllocSym = + MF.getMMI().getContext().getOrCreateFrameAllocSymbol(MF.getName()); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl, + TII->get(TargetOpcode::FRAME_ALLOC)) + .addSym(FrameAllocSym) + .addFrameIndex(Alloc); + + return nullptr; + } + + case Intrinsic::recoverframeallocation: { + // i8* @llvm.recoverframeallocation(i8* %fn, i8* %fp) + MachineFunction &MF = DAG.getMachineFunction(); + MVT PtrVT = TLI.getPointerTy(0); + + // Get the symbol that defines the frame offset. + Function *Fn = cast(I.getArgOperand(0)->stripPointerCasts()); + MCSymbol *FrameAllocSym = + MF.getMMI().getContext().getOrCreateFrameAllocSymbol(Fn->getName()); + + // Create a TargetExternalSymbol for the label to avoid any target lowering + // that would make this PC relative. + StringRef Name = FrameAllocSym->getName(); + assert(Name.size() == strlen(Name.data()) && "not null terminated"); + SDValue OffsetSym = DAG.getTargetExternalSymbol(Name.data(), PtrVT); + SDValue OffsetVal = + DAG.getNode(ISD::RECOVER_FRAME_ALLOC, sdl, PtrVT, OffsetSym); + + // Add the offset to the FP. + Value *FP = I.getArgOperand(1); + SDValue FPVal = getValue(FP); + SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal); + setValue(&I, Add); + + return nullptr; + } } } Index: llvm/trunk/lib/IR/Verifier.cpp =================================================================== --- llvm/trunk/lib/IR/Verifier.cpp +++ llvm/trunk/lib/IR/Verifier.cpp @@ -198,9 +198,14 @@ /// personality function. const Value *PersonalityFn; + /// \brief Whether we've seen a call to @llvm.frameallocate in this function + /// already. + bool SawFrameAllocate; + public: explicit Verifier(raw_ostream &OS = dbgs()) - : VerifierSupport(OS), Context(nullptr), PersonalityFn(nullptr) {} + : VerifierSupport(OS), Context(nullptr), PersonalityFn(nullptr), + SawFrameAllocate(false) {} bool verify(const Function &F) { M = F.getParent(); @@ -235,6 +240,7 @@ visit(const_cast(F)); InstsInThisBlock.clear(); PersonalityFn = nullptr; + SawFrameAllocate = false; return !Broken; } @@ -2599,7 +2605,26 @@ Assert1(isa(CI.getArgOperand(1)), "llvm.invariant.end parameter #2 must be a constant integer", &CI); break; - + + case Intrinsic::frameallocate: { + BasicBlock *BB = CI.getParent(); + Assert1(BB == &BB->getParent()->front(), + "llvm.frameallocate used outside of entry block", &CI); + Assert1(!SawFrameAllocate, + "multiple calls to llvm.frameallocate in one function", &CI); + SawFrameAllocate = true; + Assert1(isa(CI.getArgOperand(0)), + "llvm.frameallocate argument must be constant integer size", &CI); + break; + } + case Intrinsic::recoverframeallocation: { + Value *FnArg = CI.getArgOperand(0)->stripPointerCasts(); + Function *Fn = dyn_cast(FnArg); + Assert1(Fn && !Fn->isDeclaration(), "llvm.recoverframeallocation first " + "argument must be function defined in this module", &CI); + break; + } + case Intrinsic::experimental_gc_statepoint: { Assert1(!CI.doesNotAccessMemory() && !CI.onlyReadsMemory(), Index: llvm/trunk/lib/MC/MCContext.cpp =================================================================== --- llvm/trunk/lib/MC/MCContext.cpp +++ llvm/trunk/lib/MC/MCContext.cpp @@ -130,6 +130,11 @@ return Sym; } +MCSymbol *MCContext::getOrCreateFrameAllocSymbol(StringRef FuncName) { + return GetOrCreateSymbol(Twine(MAI->getPrivateGlobalPrefix()) + + "frameallocation_" + FuncName); +} + MCSymbol *MCContext::CreateSymbol(StringRef Name) { // Determine whether this is an assembler temporary or normal label, if used. bool isTemporary = false; Index: llvm/trunk/lib/Target/X86/X86InstrCompiler.td =================================================================== --- llvm/trunk/lib/Target/X86/X86InstrCompiler.td +++ llvm/trunk/lib/Target/X86/X86InstrCompiler.td @@ -1010,6 +1010,9 @@ (MOV64mi32 addr:$dst, tblockaddress:$src)>, Requires<[NearData, IsStatic]>; +def : Pat<(i32 (X86RecoverFrameAlloc texternalsym:$dst)), (MOV32ri texternalsym:$dst)>; +def : Pat<(i64 (X86RecoverFrameAlloc texternalsym:$dst)), (MOV64ri texternalsym:$dst)>; + // Calls // tls has some funny stuff here... Index: llvm/trunk/lib/Target/X86/X86InstrInfo.td =================================================================== --- llvm/trunk/lib/Target/X86/X86InstrInfo.td +++ llvm/trunk/lib/Target/X86/X86InstrInfo.td @@ -194,6 +194,10 @@ def X86Wrapper : SDNode<"X86ISD::Wrapper", SDTX86Wrapper>; def X86WrapperRIP : SDNode<"X86ISD::WrapperRIP", SDTX86Wrapper>; +def X86RecoverFrameAlloc : SDNode<"ISD::RECOVER_FRAME_ALLOC", + SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>, + SDTCisInt<1>]>>; + def X86tlsaddr : SDNode<"X86ISD::TLSADDR", SDT_X86TLSADDR, [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>; Index: llvm/trunk/test/CodeGen/X86/frameallocate.ll =================================================================== --- llvm/trunk/test/CodeGen/X86/frameallocate.ll +++ llvm/trunk/test/CodeGen/X86/frameallocate.ll @@ -0,0 +1,39 @@ +; RUN: llc -mtriple=x86_64-windows-msvc < %s | FileCheck %s + +declare i8* @llvm.frameallocate(i32) +declare i8* @llvm.frameaddress(i32) +declare i8* @llvm.recoverframeallocation(i8*, i8*) +declare i32 @printf(i8*, ...) + +@str = internal constant [10 x i8] c"asdf: %d\0A\00" + +define void @print_framealloc_from_fp(i8* %fp) { + %alloc = call i8* @llvm.recoverframeallocation(i8* bitcast (void(i32*, i32*)* @alloc_func to i8*), i8* %fp) + %alloc_i32 = bitcast i8* %alloc to i32* + %r = load i32* %alloc_i32 + call i32 (i8*, ...)* @printf(i8* getelementptr ([10 x i8]* @str, i32 0, i32 0), i32 %r) + ret void +} + +; CHECK-LABEL: print_framealloc_from_fp: +; CHECK: movabsq $.Lframeallocation_alloc_func, %[[offs:[a-z]+]] +; CHECK: movl (%rcx,%[[offs]]), %edx +; CHECK: leaq {{.*}}(%rip), %rcx +; CHECK: callq printf +; CHECK: retq + +define void @alloc_func(i32* %s, i32* %d) { + %alloc = call i8* @llvm.frameallocate(i32 16) + %alloc_i32 = bitcast i8* %alloc to i32* + store i32 42, i32* %alloc_i32 + %fp = call i8* @llvm.frameaddress(i32 0) + call void @print_framealloc_from_fp(i8* %fp) + ret void +} + +; CHECK-LABEL: alloc_func: +; CHECK: .Lframeallocation_alloc_func = -[[offs:[0-9]+]] +; CHECK: movl $42, -[[offs]](%rbp) +; CHECK: movq %rbp, %rcx +; CHECK: callq print_framealloc_from_fp +; CHECK: retq Index: llvm/trunk/test/Verifier/frameallocate.ll =================================================================== --- llvm/trunk/test/Verifier/frameallocate.ll +++ llvm/trunk/test/Verifier/frameallocate.ll @@ -0,0 +1,48 @@ +; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s + +declare i8* @llvm.frameallocate(i32) +declare i8* @llvm.recoverframeallocation(i8*, i8*) + +define internal void @f() { + call i8* @llvm.frameallocate(i32 4) + call i8* @llvm.frameallocate(i32 4) + ret void +} +; CHECK: multiple calls to llvm.frameallocate in one function + +define internal void @f_a(i32 %n) { + call i8* @llvm.frameallocate(i32 %n) + ret void +} +; CHECK: llvm.frameallocate argument must be constant integer size + +define internal void @g() { +entry: + br label %not_entry +not_entry: + call i8* @llvm.frameallocate(i32 4) + ret void +} +; CHECK: llvm.frameallocate used outside of entry block + +define internal void @h() { + call i8* @llvm.recoverframeallocation(i8* null, i8* null) + ret void +} +; CHECK: llvm.recoverframeallocation first argument must be function defined in this module + +@global = constant i8 0 + +declare void @declaration() + +define internal void @i() { + call i8* @llvm.recoverframeallocation(i8* @global, i8* null) + ret void +} +; CHECK: llvm.recoverframeallocation first argument must be function defined in this module + +define internal void @j() { + call i8* @llvm.recoverframeallocation(i8* bitcast(void()* @declaration to i8*), i8* null) + ret void +} +; CHECK: llvm.recoverframeallocation first argument must be function defined in this module Index: llvm/trunk/utils/TableGen/CodeGenTarget.cpp =================================================================== --- llvm/trunk/utils/TableGen/CodeGenTarget.cpp +++ llvm/trunk/utils/TableGen/CodeGenTarget.cpp @@ -295,7 +295,7 @@ "IMPLICIT_DEF", "SUBREG_TO_REG", "COPY_TO_REGCLASS", "DBG_VALUE", "REG_SEQUENCE", "COPY", "BUNDLE", "LIFETIME_START", "LIFETIME_END", "STACKMAP", "PATCHPOINT", "LOAD_STACK_GUARD", - "STATEPOINT", + "STATEPOINT", "FRAME_ALLOC", nullptr}; const auto &Insts = getInstructions(); for (const char *const *p = FixedInstrs; *p; ++p) {