Index: include/llvm/CodeGen/AsmPrinter.h =================================================================== --- include/llvm/CodeGen/AsmPrinter.h +++ include/llvm/CodeGen/AsmPrinter.h @@ -49,6 +49,7 @@ class GlobalValue; class GlobalVariable; class MachineBasicBlock; +class MachineConstantPoolEntry; class MachineConstantPoolValue; class MachineDominatorTree; class MachineFunction; @@ -137,6 +138,9 @@ static char ID; + /// Sections that need to be referenced in the POT + std::vector POT; + struct HandlerInfo { AsmPrinterHandler *Handler; const char *TimerName; @@ -231,6 +235,16 @@ MCSymbol *getSymbol(const GlobalValue *GV) const; + /// Return the section this constant pool entry should be emitted to. Update + /// Align with the section's alignment. + MCSection *getSectionForCPE(const MachineConstantPoolEntry &CPE, + unsigned &Align) const; + + /// Lookup the constant pool entry corresponding to the given constant pool + /// index for the current function and return the section to emit this entry + /// to. Update Align with the section's alignment. + MCSection *getSectionForCPI(unsigned CPID, unsigned &Align) const; + //===------------------------------------------------------------------===// // XRay instrumentation implementation. //===------------------------------------------------------------------===// @@ -451,6 +465,16 @@ MCSymbol *GetBlockAddressSymbol(const BlockAddress *BA) const; MCSymbol *GetBlockAddressSymbol(const BasicBlock *BB) const; + /// Return the MCSymbol for the start of the section containing this global + /// object, if available. + MCSymbol *GetSectionSymbol(const GlobalObject *GO) const; + MCSymbol *GetSectionSymbol(unsigned CPID) const; + + /// Return the POT index of the section containing this global object. + unsigned GetPOTIndex(const GlobalObject *GO); + /// Return the POT index of the section containing this constant pool ID. + unsigned GetPOTIndex(unsigned CPID); + //===------------------------------------------------------------------===// // Emission Helper Routines. //===------------------------------------------------------------------===// @@ -649,6 +673,7 @@ void EmitJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid) const; + void EmitPOT(); void EmitLLVMUsedList(const ConstantArray *InitList); /// Emit llvm.ident metadata in an '.ident' directive. void EmitModuleIdents(Module &M); @@ -660,6 +685,8 @@ void emitGlobalIndirectSymbol(Module &M, const GlobalIndirectSymbol &GIS); void setupCodePaddingContext(const MachineBasicBlock &MBB, MCCodePaddingContext &Context) const; + /// Return the POT index of the section. + unsigned GetPOTIndex(const MCSection *Sec); }; } // end namespace llvm Index: include/llvm/CodeGen/ISDOpcodes.h =================================================================== --- include/llvm/CodeGen/ISDOpcodes.h +++ include/llvm/CodeGen/ISDOpcodes.h @@ -65,6 +65,9 @@ /// The address of the GOT GLOBAL_OFFSET_TABLE, + /// The address of the POT + PAGE_OFFSET_TABLE, + /// FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and /// llvm.returnaddress on the DAG. These nodes take one operand, the index /// of the frame or return address to return. An index of zero corresponds Index: include/llvm/CodeGen/MachineMemOperand.h =================================================================== --- include/llvm/CodeGen/MachineMemOperand.h +++ include/llvm/CodeGen/MachineMemOperand.h @@ -109,6 +109,10 @@ /// Return a MachinePointerInfo record that refers to a GOT entry. static MachinePointerInfo getGOT(MachineFunction &MF); + /// Return a MachinePointerInfo record that refers to a Page Offset Table + /// (POT) entry. + static MachinePointerInfo getPOT(MachineFunction &MF); + /// Stack pointer relative access. static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID = 0); Index: include/llvm/CodeGen/PseudoSourceValue.h =================================================================== --- include/llvm/CodeGen/PseudoSourceValue.h +++ include/llvm/CodeGen/PseudoSourceValue.h @@ -39,6 +39,7 @@ enum PSVKind : unsigned { Stack, GOT, + POT, JumpTable, ConstantPool, FixedStack, @@ -68,6 +69,7 @@ bool isStack() const { return Kind == Stack; } bool isGOT() const { return Kind == GOT; } + bool isPOT() const { return Kind == POT; } bool isConstantPool() const { return Kind == ConstantPool; } bool isJumpTable() const { return Kind == JumpTable; } @@ -156,7 +158,7 @@ /// Manages creation of pseudo source values. class PseudoSourceValueManager { const TargetInstrInfo &TII; - const PseudoSourceValue StackPSV, GOTPSV, JumpTablePSV, ConstantPoolPSV; + const PseudoSourceValue StackPSV, GOTPSV, POTPSV, JumpTablePSV, ConstantPoolPSV; std::map> FSValues; StringMap> ExternalCallEntries; @@ -175,6 +177,9 @@ /// (or something the like). const PseudoSourceValue *getGOT(); + /// Return a pseudo source value referencing the POT. + const PseudoSourceValue *getPOT(); + /// Return a pseudo source value referencing the constant pool. Since constant /// pools are constant, this doesn't need to identify a specific constant /// pool entry. Index: include/llvm/CodeGen/TargetLowering.h =================================================================== --- include/llvm/CodeGen/TargetLowering.h +++ include/llvm/CodeGen/TargetLowering.h @@ -1404,6 +1404,12 @@ report_fatal_error("Funclet EH is not implemented for this target"); } + /// If a physical register, this returns the register that stores the base + /// address of the POT. + unsigned getPOTBaseRegister() const { + return POTBaseRegister; + } + /// Returns the target's jmp_buf size in bytes (if never set, the default is /// 200) unsigned getJumpBufSize() const { @@ -1785,6 +1791,12 @@ StackPointerRegisterToSaveRestore = R; } + /// If set to a physical register, this sets the register that stores the POT + /// base address. + void setPOTBaseRegister(unsigned R) { + POTBaseRegister = R; + } + /// Tells the code generator that the target has multiple (allocatable) /// condition registers that can be used to store the results of comparisons /// for use by selects and conditional branches. With multiple condition @@ -2480,6 +2492,10 @@ /// llvm.savestack/llvm.restorestack should save and restore. unsigned StackPointerRegisterToSaveRestore; + /// If set to a physical register, this specifies the register that + /// stores the POT base register. + unsigned POTBaseRegister; + /// This indicates the default register class to use for each ValueType the /// target supports natively. const TargetRegisterClass *RegClassForVT[MVT::LAST_VALUETYPE]; Index: include/llvm/CodeGen/TargetLoweringObjectFileImpl.h =================================================================== --- include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -90,6 +90,10 @@ const MCExpr *lowerRelativeReference(const GlobalValue *LHS, const GlobalValue *RHS, const TargetMachine &TM) const override; + + /// Return a symbol that refers to the base of the GOT. Platforms that do not + /// refer to the GOT as _GLOBAL_OFFSET_TABLE_ should override this function. + virtual MCSymbol *getGlobalOffsetTableSymbol() const override; }; class TargetLoweringObjectFileMachO : public TargetLoweringObjectFile { Index: include/llvm/Target/TargetLoweringObjectFile.h =================================================================== --- include/llvm/Target/TargetLoweringObjectFile.h +++ include/llvm/Target/TargetLoweringObjectFile.h @@ -201,6 +201,10 @@ virtual void emitLinkerFlagsForUsed(raw_ostream &OS, const GlobalValue *GV) const {} + virtual MCSymbol *getGlobalOffsetTableSymbol() const { + return nullptr; + } + protected: virtual MCSection *SelectSectionForGlobal(const GlobalObject *GO, SectionKind Kind, Index: lib/CodeGen/AsmPrinter/AsmPrinter.cpp =================================================================== --- lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -429,6 +429,29 @@ return TM.getSymbol(GV); } +MCSection *AsmPrinter::getSectionForCPE(const MachineConstantPoolEntry &CPE, + unsigned &Align) const { + // getSectionForConstant() may update the alignment depending on the target + // format. We must respect this update and propagate it back to our caller. + Align = CPE.getAlignment(); + + SectionKind Kind = CPE.getSectionKind(&getDataLayout()); + + const Constant *C = nullptr; + if (!CPE.isMachineConstantPoolEntry()) + C = CPE.Val.ConstVal; + + return getObjFileLowering().getSectionForConstant(getDataLayout(), + Kind, C, Align); +} + +MCSection *AsmPrinter::getSectionForCPI(unsigned CPID, unsigned &Align) const { + const MachineConstantPool *MCP = MF->getConstantPool(); + const MachineConstantPoolEntry &CPE = MCP->getConstants()[CPID]; + + return getSectionForCPE(CPE, Align); +} + /// EmitGlobalVariable - Emit the specified global variable to the .s file. void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { bool IsEmuTLSVar = TM.useEmulatedTLS() && GV->isThreadLocal(); @@ -1381,6 +1404,10 @@ // Emit remaining GOT equivalent globals. emitGlobalGOTEquivs(); + // Emit POT (page offset table) if Pagerando is enabled. + if (TM.isPagerando()) + EmitPOT(); + // Emit visibility info for declarations for (const Function &F : M) { if (!F.isDeclarationForLinker()) @@ -1658,16 +1685,8 @@ SmallVector CPSections; for (unsigned i = 0, e = CP.size(); i != e; ++i) { const MachineConstantPoolEntry &CPE = CP[i]; - unsigned Align = CPE.getAlignment(); - - SectionKind Kind = CPE.getSectionKind(&getDataLayout()); - - const Constant *C = nullptr; - if (!CPE.isMachineConstantPoolEntry()) - C = CPE.Val.ConstVal; - - MCSection *S = getObjFileLowering().getSectionForConstant(getDataLayout(), - Kind, C, Align); + unsigned Align; + MCSection *S = getSectionForCPE(CPE, Align); // The number of sections are small, just do a linear search from the // last section to the first. @@ -1900,6 +1919,46 @@ report_fatal_error("unknown special variable"); } +/// EmitPOT - Emit the page offset table (POT), used for inter-page references +/// in pagerando. Pagerando allows specially marked code pages (bins) in a DSO +/// to be loaded at independent offsets from each other and the rest of the +/// DSO. The POT stores the dynamic absolute address of each pagerando section +/// to allow references to these pagerando sections from code that is not +/// located at a static PC-relative offset from the target section. The first +/// entry in the POT stores the absolute address of the global offset table +/// (GOT) to allow code in pagerando sections to reference global data and +/// non-pagerando code with GOT-relative relocations. +void AsmPrinter::EmitPOT() { + assert(TM.getTargetTriple().isOSBinFormatELF() && + "Pagerando is only supported on ELF"); + unsigned Alignment = getDataLayout().getPointerPrefAlignment(0); + unsigned PtrSize = getDataLayout().getPointerSize(0); + + auto *Section = OutContext.getELFSection(".pot", ELF::SHT_PROGBITS, + ELF::SHF_ALLOC | ELF::SHF_WRITE); + OutStreamer->SwitchSection(Section); + + // Emit POT start label + auto *POTSym = OutContext.getOrCreateSymbol("_PAGE_OFFSET_TABLE_"); + OutStreamer->EmitValueToAlignment(Alignment); + OutStreamer->EmitLabel(POTSym); + + // Entry 0 is GOT reference + auto *GOTSym = getObjFileLowering().getGlobalOffsetTableSymbol(); + assert(GOTSym && "Target platform must support absolute GOT symbol"); + EmitLabelReference(GOTSym, PtrSize); + + // Emit Bin references + for (auto *Bin : POT) + EmitLabelReference(Bin->getBeginSymbol(), PtrSize); + + // Emit POT end label + auto *POTEndSym = OutContext.getOrCreateSymbol("_PAGE_OFFSET_TABLE_END_"); + OutStreamer->EmitLabel(POTEndSym); + + OutStreamer->AddBlankLine(); +} + /// EmitLLVMUsedList - For targets that define a MAI::UsedDirective, mark each /// global in the specified llvm.used list for which emitUsedDirectiveFor /// is true, as being used with this directive. @@ -2764,6 +2823,36 @@ return OutContext.getOrCreateSymbol(NameStr); } +MCSymbol *AsmPrinter::GetSectionSymbol(const GlobalObject *GO) const { + MCSection *Sec = getObjFileLowering().SectionForGlobal(GO, TM); + return Sec->getBeginSymbol(); +} + +MCSymbol *AsmPrinter::GetSectionSymbol(unsigned CPID) const { + unsigned ignored; + MCSection *Sec = getSectionForCPI(CPID, ignored); + return Sec->getBeginSymbol(); +} + +unsigned AsmPrinter::GetPOTIndex(const GlobalObject *GO) { + const MCSection *Sec = getObjFileLowering().SectionForGlobal(GO, TM); + return GetPOTIndex(Sec); +} + +unsigned AsmPrinter::GetPOTIndex(unsigned CPID) { + unsigned ignored; + const MCSection *Sec = getSectionForCPI(CPID, ignored); + return GetPOTIndex(Sec); +} + +unsigned AsmPrinter::GetPOTIndex(const MCSection *Sec) { + auto I = std::find(POT.begin(), POT.end(), Sec); + auto Index = static_cast(I - POT.begin()); + if (I == POT.end()) + POT.push_back(Sec); + return Index + 1; // Index 0 denotes the default bin #0 +} + /// PrintParentLoopComment - Print comments about parent loops of this one. static void PrintParentLoopComment(raw_ostream &OS, const MachineLoop *Loop, unsigned FunctionNumber) { Index: lib/CodeGen/MachineOperand.cpp =================================================================== --- lib/CodeGen/MachineOperand.cpp +++ lib/CodeGen/MachineOperand.cpp @@ -968,6 +968,10 @@ return MachinePointerInfo(MF.getPSVManager().getGOT()); } +MachinePointerInfo MachinePointerInfo::getPOT(MachineFunction &MF) { + return MachinePointerInfo(MF.getPSVManager().getPOT()); +} + MachinePointerInfo MachinePointerInfo::getStack(MachineFunction &MF, int64_t Offset, uint8_t ID) { return MachinePointerInfo(MF.getPSVManager().getStack(), Offset, ID); @@ -1097,6 +1101,9 @@ case PseudoSourceValue::GOT: OS << "got"; break; + case PseudoSourceValue::POT: + OS << "pot"; + break; case PseudoSourceValue::JumpTable: OS << "jump-table"; break; Index: lib/CodeGen/PseudoSourceValue.cpp =================================================================== --- lib/CodeGen/PseudoSourceValue.cpp +++ lib/CodeGen/PseudoSourceValue.cpp @@ -22,7 +22,7 @@ using namespace llvm; static const char *const PSVNames[] = { - "Stack", "GOT", "JumpTable", "ConstantPool", "FixedStack", + "Stack", "GOT", "POT", "JumpTable", "ConstantPool", "FixedStack", "GlobalValueCallEntry", "ExternalSymbolCallEntry"}; PseudoSourceValue::PseudoSourceValue(unsigned Kind, const TargetInstrInfo &TII) @@ -43,19 +43,19 @@ bool PseudoSourceValue::isConstant(const MachineFrameInfo *) const { if (isStack()) return false; - if (isGOT() || isConstantPool() || isJumpTable()) + if (isGOT() || isPOT() || isConstantPool() || isJumpTable()) return true; llvm_unreachable("Unknown PseudoSourceValue!"); } bool PseudoSourceValue::isAliased(const MachineFrameInfo *) const { - if (isStack() || isGOT() || isConstantPool() || isJumpTable()) + if (isStack() || isGOT() || isPOT() || isConstantPool() || isJumpTable()) return false; llvm_unreachable("Unknown PseudoSourceValue!"); } bool PseudoSourceValue::mayAlias(const MachineFrameInfo *) const { - return !(isGOT() || isConstantPool() || isJumpTable()); + return !(isGOT() || isPOT() || isConstantPool() || isJumpTable()); } bool FixedStackPseudoSourceValue::isConstant( @@ -109,6 +109,7 @@ : TII(TIInfo), StackPSV(PseudoSourceValue::Stack, TII), GOTPSV(PseudoSourceValue::GOT, TII), + POTPSV(PseudoSourceValue::POT, TII), JumpTablePSV(PseudoSourceValue::JumpTable, TII), ConstantPoolPSV(PseudoSourceValue::ConstantPool, TII) {} @@ -118,6 +119,8 @@ const PseudoSourceValue *PseudoSourceValueManager::getGOT() { return &GOTPSV; } +const PseudoSourceValue *PseudoSourceValueManager::getPOT() { return &POTPSV; } + const PseudoSourceValue *PseudoSourceValueManager::getConstantPool() { return &ConstantPoolPSV; } Index: lib/CodeGen/SelectionDAG/LegalizeDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -3942,6 +3942,7 @@ } case ISD::GLOBAL_OFFSET_TABLE: + case ISD::PAGE_OFFSET_TABLE: case ISD::GlobalAddress: case ISD::GlobalTLSAddress: case ISD::ExternalSymbol: Index: lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -121,6 +121,7 @@ case ISD::FrameIndex: return "FrameIndex"; case ISD::JumpTable: return "JumpTable"; case ISD::GLOBAL_OFFSET_TABLE: return "GLOBAL_OFFSET_TABLE"; + case ISD::PAGE_OFFSET_TABLE: return "PAGE_OFFSET_TABLE"; case ISD::RETURNADDR: return "RETURNADDR"; case ISD::ADDROFRETURNADDR: return "ADDROFRETURNADDR"; case ISD::FRAMEADDR: return "FRAMEADDR"; Index: lib/CodeGen/TargetLoweringBase.cpp =================================================================== --- lib/CodeGen/TargetLoweringBase.cpp +++ lib/CodeGen/TargetLoweringBase.cpp @@ -548,6 +548,7 @@ EnableExtLdPromotion = false; HasFloatingPointExceptions = true; StackPointerRegisterToSaveRestore = 0; + POTBaseRegister = 0; BooleanContents = UndefinedBooleanContent; BooleanFloatContents = UndefinedBooleanContent; BooleanVectorContents = UndefinedBooleanContent; Index: lib/CodeGen/TargetLoweringObjectFileImpl.cpp =================================================================== --- lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -676,6 +676,12 @@ Flags |= ELF::SHF_LINK_ORDER; } + // Pagerando is incompatible with unique sections. + if (auto *F = dyn_cast(GO)) { + if (F->isPagerando()) + EmitUniqueSection = false; + } + MCSectionELF *Section = selectELFSectionForGlobal( getContext(), GO, Kind, getMangler(), TM, EmitUniqueSection, Flags, &NextUniqueID, AssociatedSymbol); @@ -700,6 +706,11 @@ bool TargetLoweringObjectFileELF::shouldPutJumpTableInFunctionSection( bool UsesLabelDifference, const Function &F) const { + // If we place the function in a randomly located page, it's faster to have a + // local jump table, since a global one would require dynamic relocations. + if (F.isPagerando()) + return true; + // We can always create relative relocations, so use another section // that can be marked non-executable. return false; @@ -795,6 +806,10 @@ MCSymbolRefExpr::create(TM.getSymbol(RHS), getContext()), getContext()); } +MCSymbol *TargetLoweringObjectFileELF::getGlobalOffsetTableSymbol() const { + return getContext().getOrCreateSymbol("_GLOBAL_OFFSET_TABLE_"); +} + void TargetLoweringObjectFileELF::InitializeELF(bool UseInitArray_) { UseInitArray = UseInitArray_; Index: lib/Target/TargetLoweringObjectFile.cpp =================================================================== --- lib/Target/TargetLoweringObjectFile.cpp +++ lib/Target/TargetLoweringObjectFile.cpp @@ -287,7 +287,15 @@ // in discardable section // FIXME: this isn't the right predicate, should be based on the MCSection // for the function. - return F.isWeakForLinker(); + if (F.isWeakForLinker()) + return true; + + // If we place the function in a randomly located page, it's faster to have a + // local jump table, since a global one would require dynamic relocations. + if (F.isPagerando()) + return true; + + return false; } /// Given a mergable constant with the specified size and relocation