Index: include/llvm/CodeGen/AsmPrinter.h =================================================================== --- include/llvm/CodeGen/AsmPrinter.h +++ include/llvm/CodeGen/AsmPrinter.h @@ -135,6 +135,9 @@ /// If VerboseAsm is set, a pointer to the loop info for this function. MachineLoopInfo *LI = nullptr; + /// Sections that need to be referenced in the POT + std::vector POT; + struct HandlerInfo { AsmPrinterHandler *Handler; const char *TimerName; @@ -220,6 +223,8 @@ MCSymbol *getSymbol(const GlobalValue *GV) const; + MCSection *getSectionForCPI(unsigned CPID) const; + //===------------------------------------------------------------------===// // XRay instrumentation implementation. //===------------------------------------------------------------------===// @@ -441,6 +446,16 @@ MCSymbol *GetBlockAddressSymbol(const BlockAddress *BA) const; MCSymbol *GetBlockAddressSymbol(const BasicBlock *BB) const; + /// Return the MCSymbol for the start of the section containing this global + /// object, if available. + MCSymbol *GetSectionSymbol(const GlobalObject *GO) const; + MCSymbol *GetSectionSymbol(unsigned CPID) const; + + /// Return the POT index of the section containing this global object. + unsigned GetPOTIndex(const GlobalObject *GO); + /// Return the POT index of the section containing this constant pool ID. + unsigned GetPOTIndex(unsigned CPID); + //===------------------------------------------------------------------===// // Emission Helper Routines. //===------------------------------------------------------------------===// @@ -623,6 +638,7 @@ void EmitJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid) const; + void EmitPOT(); void EmitLLVMUsedList(const ConstantArray *InitList); /// Emit llvm.ident metadata in an '.ident' directive. void EmitModuleIdents(Module &M); @@ -630,8 +646,9 @@ bool isCtor); GCMetadataPrinter *GetOrCreateGCPrinter(GCStrategy &C); /// Emit GlobalAlias or GlobalIFunc. - void emitGlobalIndirectSymbol(Module &M, - const GlobalIndirectSymbol& GIS); + void emitGlobalIndirectSymbol(Module &M, const GlobalIndirectSymbol &GIS); + /// Return the POT index of the section. + unsigned GetPOTIndex(const MCSection *Sec); }; } // end namespace llvm Index: include/llvm/CodeGen/ISDOpcodes.h =================================================================== --- include/llvm/CodeGen/ISDOpcodes.h +++ include/llvm/CodeGen/ISDOpcodes.h @@ -65,6 +65,9 @@ /// The address of the GOT GLOBAL_OFFSET_TABLE, + /// The address of the POT + PAGE_OFFSET_TABLE, + /// FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and /// llvm.returnaddress on the DAG. These nodes take one operand, the index /// of the frame or return address to return. An index of zero corresponds Index: include/llvm/CodeGen/MachineMemOperand.h =================================================================== --- include/llvm/CodeGen/MachineMemOperand.h +++ include/llvm/CodeGen/MachineMemOperand.h @@ -86,6 +86,10 @@ /// Return a MachinePointerInfo record that refers to a GOT entry. static MachinePointerInfo getGOT(MachineFunction &MF); + /// Return a MachinePointerInfo record that refers to a Page Offset Table + /// (POT) entry. + static MachinePointerInfo getPOT(MachineFunction &MF); + /// Stack pointer relative access. static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID = 0); Index: include/llvm/CodeGen/PseudoSourceValue.h =================================================================== --- include/llvm/CodeGen/PseudoSourceValue.h +++ include/llvm/CodeGen/PseudoSourceValue.h @@ -38,6 +38,7 @@ enum PSVKind { Stack, GOT, + POT, JumpTable, ConstantPool, FixedStack, @@ -66,6 +67,7 @@ bool isStack() const { return Kind == Stack; } bool isGOT() const { return Kind == GOT; } + bool isPOT() const { return Kind == POT; } bool isConstantPool() const { return Kind == ConstantPool; } bool isJumpTable() const { return Kind == JumpTable; } unsigned getTargetCustom() const { @@ -149,7 +151,7 @@ /// Manages creation of pseudo source values. class PseudoSourceValueManager { - const PseudoSourceValue StackPSV, GOTPSV, JumpTablePSV, ConstantPoolPSV; + const PseudoSourceValue StackPSV, GOTPSV, POTPSV, JumpTablePSV, ConstantPoolPSV; std::map> FSValues; StringMap> ExternalCallEntries; @@ -168,6 +170,9 @@ /// (or something the like). const PseudoSourceValue *getGOT(); + /// Return a pseudo source value referencing the POT. + const PseudoSourceValue *getPOT(); + /// Return a pseudo source value referencing the constant pool. Since constant /// pools are constant, this doesn't need to identify a specific constant /// pool entry. Index: include/llvm/Target/TargetLowering.h =================================================================== --- include/llvm/Target/TargetLowering.h +++ include/llvm/Target/TargetLowering.h @@ -1312,6 +1312,12 @@ report_fatal_error("Funclet EH is not implemented for this target"); } + /// If a physical register, this returns the register that stores the base + /// address of the POT. + unsigned getPOTBaseRegister() const { + return POTBaseRegister; + } + /// Returns the target's jmp_buf size in bytes (if never set, the default is /// 200) unsigned getJumpBufSize() const { @@ -1649,6 +1655,12 @@ StackPointerRegisterToSaveRestore = R; } + /// If set to a physical register, this sets the register that stores the POT + /// base address. + void setPOTBaseRegister(unsigned R) { + POTBaseRegister = R; + } + /// Tells the code generator that the target has multiple (allocatable) /// condition registers that can be used to store the results of comparisons /// for use by selects and conditional branches. With multiple condition @@ -2317,6 +2329,10 @@ /// llvm.savestack/llvm.restorestack should save and restore. unsigned StackPointerRegisterToSaveRestore; + /// If set to a physical register, this specifies the register that + /// stores the POT base register. + unsigned POTBaseRegister; + /// This indicates the default register class to use for each ValueType the /// target supports natively. const TargetRegisterClass *RegClassForVT[MVT::LAST_VALUETYPE]; Index: lib/CodeGen/AsmPrinter/AsmPrinter.cpp =================================================================== --- lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -423,6 +423,23 @@ return TM.getSymbol(GV); } +MCSection *AsmPrinter::getSectionForCPI(unsigned CPID) const { + const MachineConstantPool *MCP = MF->getConstantPool(); + const MachineConstantPoolEntry &CPE = MCP->getConstants()[CPID]; + + // TODO(sjc): refactor this with redundant code in EmitConstantPool() + unsigned Align = CPE.getAlignment(); + + SectionKind Kind = CPE.getSectionKind(&getDataLayout()); + + const Constant *C = nullptr; + if (!CPE.isMachineConstantPoolEntry()) + C = CPE.Val.ConstVal; + + return getObjFileLowering().getSectionForConstant(getDataLayout(), + Kind, C, Align); +} + /// EmitGlobalVariable - Emit the specified global variable to the .s file. void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { bool IsEmuTLSVar = TM.Options.EmulatedTLS && GV->isThreadLocal(); @@ -1276,6 +1293,10 @@ // Emit remaining GOT equivalent globals. emitGlobalGOTEquivs(); + // Emit POT (page offset table) if Pagerando is enabled. + if (TM.isPagerando()) + EmitPOT(); + // Emit visibility info for declarations for (const Function &F : M) { if (!F.isDeclarationForLinker()) @@ -1460,15 +1481,7 @@ for (unsigned i = 0, e = CP.size(); i != e; ++i) { const MachineConstantPoolEntry &CPE = CP[i]; unsigned Align = CPE.getAlignment(); - - SectionKind Kind = CPE.getSectionKind(&getDataLayout()); - - const Constant *C = nullptr; - if (!CPE.isMachineConstantPoolEntry()) - C = CPE.Val.ConstVal; - - MCSection *S = getObjFileLowering().getSectionForConstant(getDataLayout(), - Kind, C, Align); + MCSection *S = getSectionForCPI(i); // The number of sections are small, just do a linear search from the // last section to the first. @@ -1702,6 +1715,37 @@ report_fatal_error("unknown special variable"); } +void AsmPrinter::EmitPOT() { + assert(TM.getTargetTriple().isOSBinFormatELF() && + "Pagerando is only supported on ELF"); + unsigned Alignment = getDataLayout().getPointerPrefAlignment(0); + unsigned PtrSize = getDataLayout().getPointerSize(0); + + auto *Section = OutContext.getELFSection(".pot", ELF::SHT_PROGBITS, + ELF::SHF_ALLOC | ELF::SHF_WRITE); + OutStreamer->SwitchSection(Section); + + // Emit POT start label + auto *POTSym = OutContext.getOrCreateSymbol("_PAGE_OFFSET_TABLE_"); + OutStreamer->EmitValueToAlignment(Alignment); + OutStreamer->EmitLabel(POTSym); + + // Entry 0 is GOT reference + auto *GOTSym = OutContext.getOrCreateSymbol("_GLOBAL_OFFSET_TABLE_"); + EmitLabelReference(GOTSym, PtrSize); + + // Emit Bin references + for (auto *Bin : POT) { + EmitLabelReference(Bin->getBeginSymbol(), PtrSize); + } + + // Emit POT end label + auto *POTEndSym = OutContext.getOrCreateSymbol("_PAGE_OFFSET_TABLE_END_"); + OutStreamer->EmitLabel(POTEndSym); + + OutStreamer->AddBlankLine(); +} + /// EmitLLVMUsedList - For targets that define a MAI::UsedDirective, mark each /// global in the specified llvm.used list for which emitUsedDirectiveFor /// is true, as being used with this directive. @@ -2541,6 +2585,35 @@ return OutContext.getOrCreateSymbol(NameStr); } +MCSymbol *AsmPrinter::GetSectionSymbol(const GlobalObject *GO) const { + MCSection *Sec = getObjFileLowering().SectionForGlobal(GO, TM); + return Sec->getBeginSymbol(); +} + +MCSymbol *AsmPrinter::GetSectionSymbol(unsigned CPID) const { + MCSection *Sec = getSectionForCPI(CPID); + return Sec->getBeginSymbol(); +} + +unsigned AsmPrinter::GetPOTIndex(const GlobalObject *GO) { + const MCSection *Sec = getObjFileLowering().SectionForGlobal(GO, TM); + return GetPOTIndex(Sec); +} + +unsigned AsmPrinter::GetPOTIndex(unsigned CPID) { + const MCSection *Sec = getSectionForCPI(CPID); + return GetPOTIndex(Sec); +} + +unsigned AsmPrinter::GetPOTIndex(const MCSection *Sec) { + auto I = std::find(POT.begin(), POT.end(), Sec); + auto Index = static_cast(I - POT.begin()); + if (I == POT.end()) { + POT.push_back(Sec); + } + return Index + 1; // Index 0 denotes the default bin #0 +} + /// PrintParentLoopComment - Print comments about parent loops of this one. static void PrintParentLoopComment(raw_ostream &OS, const MachineLoop *Loop, unsigned FunctionNumber) { Index: lib/CodeGen/MIRPrinter.cpp =================================================================== --- lib/CodeGen/MIRPrinter.cpp +++ lib/CodeGen/MIRPrinter.cpp @@ -1101,6 +1101,9 @@ case PseudoSourceValue::GOT: OS << "got"; break; + case PseudoSourceValue::POT: + OS << "pot"; + break; case PseudoSourceValue::JumpTable: OS << "jump-table"; break; Index: lib/CodeGen/MachineInstr.cpp =================================================================== --- lib/CodeGen/MachineInstr.cpp +++ lib/CodeGen/MachineInstr.cpp @@ -619,6 +619,10 @@ return MachinePointerInfo(MF.getPSVManager().getGOT()); } +MachinePointerInfo MachinePointerInfo::getPOT(MachineFunction &MF) { + return MachinePointerInfo(MF.getPSVManager().getPOT()); +} + MachinePointerInfo MachinePointerInfo::getStack(MachineFunction &MF, int64_t Offset, uint8_t ID) { Index: lib/CodeGen/PseudoSourceValue.cpp =================================================================== --- lib/CodeGen/PseudoSourceValue.cpp +++ lib/CodeGen/PseudoSourceValue.cpp @@ -21,7 +21,7 @@ using namespace llvm; static const char *const PSVNames[] = { - "Stack", "GOT", "JumpTable", "ConstantPool", "FixedStack", + "Stack", "GOT", "POT", "JumpTable", "ConstantPool", "FixedStack", "GlobalValueCallEntry", "ExternalSymbolCallEntry"}; PseudoSourceValue::PseudoSourceValue(PSVKind Kind) : Kind(Kind) {} @@ -38,19 +38,19 @@ bool PseudoSourceValue::isConstant(const MachineFrameInfo *) const { if (isStack()) return false; - if (isGOT() || isConstantPool() || isJumpTable()) + if (isGOT() || isPOT() || isConstantPool() || isJumpTable()) return true; llvm_unreachable("Unknown PseudoSourceValue!"); } bool PseudoSourceValue::isAliased(const MachineFrameInfo *) const { - if (isStack() || isGOT() || isConstantPool() || isJumpTable()) + if (isStack() || isGOT() || isPOT() || isConstantPool() || isJumpTable()) return false; llvm_unreachable("Unknown PseudoSourceValue!"); } bool PseudoSourceValue::mayAlias(const MachineFrameInfo *) const { - return !(isGOT() || isConstantPool() || isJumpTable()); + return !(isGOT() || isPOT() || isConstantPool() || isJumpTable()); } bool FixedStackPseudoSourceValue::isConstant( @@ -99,6 +99,7 @@ PseudoSourceValueManager::PseudoSourceValueManager() : StackPSV(PseudoSourceValue::Stack), GOTPSV(PseudoSourceValue::GOT), + POTPSV(PseudoSourceValue::POT), JumpTablePSV(PseudoSourceValue::JumpTable), ConstantPoolPSV(PseudoSourceValue::ConstantPool) {} @@ -108,6 +109,8 @@ const PseudoSourceValue *PseudoSourceValueManager::getGOT() { return &GOTPSV; } +const PseudoSourceValue *PseudoSourceValueManager::getPOT() { return &POTPSV; } + const PseudoSourceValue *PseudoSourceValueManager::getConstantPool() { return &ConstantPoolPSV; } Index: lib/CodeGen/SelectionDAG/LegalizeDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -3859,6 +3859,7 @@ break; } case ISD::GLOBAL_OFFSET_TABLE: + case ISD::PAGE_OFFSET_TABLE: case ISD::GlobalAddress: case ISD::GlobalTLSAddress: case ISD::ExternalSymbol: Index: lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -99,6 +99,7 @@ case ISD::FrameIndex: return "FrameIndex"; case ISD::JumpTable: return "JumpTable"; case ISD::GLOBAL_OFFSET_TABLE: return "GLOBAL_OFFSET_TABLE"; + case ISD::PAGE_OFFSET_TABLE: return "PAGE_OFFSET_TABLE"; case ISD::RETURNADDR: return "RETURNADDR"; case ISD::ADDROFRETURNADDR: return "ADDROFRETURNADDR"; case ISD::FRAMEADDR: return "FRAMEADDR"; Index: lib/CodeGen/TargetLoweringBase.cpp =================================================================== --- lib/CodeGen/TargetLoweringBase.cpp +++ lib/CodeGen/TargetLoweringBase.cpp @@ -483,6 +483,7 @@ EnableExtLdPromotion = false; HasFloatingPointExceptions = true; StackPointerRegisterToSaveRestore = 0; + POTBaseRegister = 0; BooleanContents = UndefinedBooleanContent; BooleanFloatContents = UndefinedBooleanContent; BooleanVectorContents = UndefinedBooleanContent; Index: lib/CodeGen/TargetLoweringObjectFileImpl.cpp =================================================================== --- lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -452,6 +452,12 @@ Flags |= ELF::SHF_LINK_ORDER; } + // Pagerando is incompatible with unique sections. + if (auto *F = dyn_cast(GO)) { + if (F->isPagerando()) + EmitUniqueSection = false; + } + MCSectionELF *Section = selectELFSectionForGlobal( getContext(), GO, Kind, getMangler(), TM, EmitUniqueSection, Flags, &NextUniqueID, AssociatedSymbol); @@ -476,6 +482,11 @@ bool TargetLoweringObjectFileELF::shouldPutJumpTableInFunctionSection( bool UsesLabelDifference, const Function &F) const { + // If we place the function in a randomly located page, it's faster to have a + // local jump table, since a global one would require dynamic relocations. + if (F.isPagerando()) + return true; + // We can always create relative relocations, so use another section // that can be marked non-executable. return false; Index: lib/Target/TargetLoweringObjectFile.cpp =================================================================== --- lib/Target/TargetLoweringObjectFile.cpp +++ lib/Target/TargetLoweringObjectFile.cpp @@ -278,7 +278,15 @@ // in discardable section // FIXME: this isn't the right predicate, should be based on the MCSection // for the function. - return F.isWeakForLinker(); + if (F.isWeakForLinker()) + return true; + + // If we place the function in a randomly located page, it's faster to have a + // local jump table, since a global one would require dynamic relocations. + if (F.isPagerando()) + return true; + + return false; } /// Given a mergable constant with the specified size and relocation