Add AArch64 backend support for pagerando

[rinon]

Summary

Extend lowering to support POT-indirect address computation.

All calls between different bins (executable segments) must be lowered using
page offset table (POT) indirect address computation when pagerando is
enabled. This patch adds support to AArch64IselLowering to support this
indirection. The AArch64PagerandoOptimizer pass then optimizes intra-bin calls
back to direct calls, since references within the same bin are guaranteeed to
have a fixed PC-relative offset.

For example, the direct call to foo:

BL <ga:@foo>

will be selected as the following, assuming foo is placed in a randomly located
bin:

%vreg1 = LOADpot %X20, <ga:@foo>
%vreg2 = MOVaddrBIN %vreg1, <ga:@foo>
BLR %vreg2

LOADpot is a pseudo-instruction that is lowered to a load of the POT entry for
the bin containing foo, and MOVaddrBIN is a pseudo-instruction that is lowered
to add the immediate relative offset of foo inside its bin to the base address
of the bin.

Inside pagerando bins, global addresses not in the GOT are computed with
approximately the following instruction sequence:

%vreg1 = LOADpot %X20, 0
%vreg2 = MOVaddrEXT <es:_GLOBAL_OFFSET_TABLE_>
%vreg3 = MOVaddr <ga:@global>
%vreg4 = SUBSXrr %vreg3, %vreg2
%vreg5 = ADDXrr %vreg1, %vreg4

This sequence computes the relative offset of the global from the GOT and adds
this static value to the dynamic GOT address loaded from the first POT
slot. After this sequence, %vreg5 contains the dynamic address of @global. The
subtraction in this sequence avoids the need for a new static relocation,
however if we can create a new GOT-relative relocation this sequence becomes
much simpler.

Global addresses found in the GOT are loaded in the conventional way, once the
GOT address is loaded from the first POT entry.

This patch set (D37580, D37581, D37582, D37583, D37584, D37585, D37586, D37587)
is a first draft of the pagerando implementation described in
http://lists.llvm.org/pipermail/llvm-dev/2017-June/113794.html.

[rinon]

This patch set (D37580, D37581, D37582, D37583, D37584, D37585, D37586, D37587) is a first draft of the pagerando implementation described in http://lists.llvm.org/pipermail/llvm-dev/2017-June/113794.html.

[rinon]

• Fix style nits

[rinon]

• Rebase
• Handle function aliases when checking for pagerando targets

Function aliases should be treated as functions to determine if the target is in
a pagerando bin.

[rinon]

Fix external symbol references from pagerando code.

Calls to external symbols from pagerando code need to go through the GOT, so we
force this in getPOT().

[rinon]

Rebase

[rinon]

• Don't tailcall from non-pagerando to pagerando functions
• Rebase

[rinon]

Rebase

[rinon]

Rebase

• Update AArch64 tests
• Fix OpFlags checks for new packed OpFlags
• Correctly handle null Callee in FastISel

[rinon]

• Fix AArch64 TargetFlags to accomodate MO_COFFSTUB along with pagerando flags
• Rebase

lib/Target/AArch64/AArch64.h

 FunctionPass *createAArch64LoadStoreOptimizationPass();
 FunctionPass *createAArch64VectorByElementOptPass();
 ModulePass *createAArch64PromoteConstantPass();
 FunctionPass *createAArch64ConditionOptimizerPass();
 FunctionPass *createAArch64A57FPLoadBalancing();
 FunctionPass *createAArch64A53Fix835769();
 FunctionPass *createFalkorHWPFFixPass();
 FunctionPass *createFalkorMarkStridedAccessesPass();
+FunctionPass *createAArch64PagerandoOptimizerPass();
 
 FunctionPass *createAArch64CleanupLocalDynamicTLSPass();
 
 FunctionPass *createAArch64CollectLOHPass();
 InstructionSelector *
 createAArch64InstructionSelector(const AArch64TargetMachine &,
                                  AArch64Subtarget &, AArch64RegisterBankInfo &);
 
 void initializeAArch64A53Fix835769Pass(PassRegistry&);
 void initializeAArch64A57FPLoadBalancingPass(PassRegistry&);
 void initializeAArch64AdvSIMDScalarPass(PassRegistry&);
 void initializeAArch64CollectLOHPass(PassRegistry&);
 void initializeAArch64CondBrTuningPass(PassRegistry &);
 void initializeAArch64ConditionalComparesPass(PassRegistry&);
 void initializeAArch64ConditionOptimizerPass(PassRegistry&);
 void initializeAArch64DeadRegisterDefinitionsPass(PassRegistry&);
 void initializeAArch64ExpandPseudoPass(PassRegistry&);
 void initializeAArch64LoadStoreOptPass(PassRegistry&);
 void initializeAArch64VectorByElementOptPass(PassRegistry&);
 void initializeAArch64PromoteConstantPass(PassRegistry&);
 void initializeAArch64RedundantCopyEliminationPass(PassRegistry&);
 void initializeAArch64StorePairSuppressPass(PassRegistry&);
 void initializeFalkorHWPFFixPass(PassRegistry&);
 void initializeFalkorMarkStridedAccessesLegacyPass(PassRegistry&);
+void initializeAArch64PagerandoOptimizerPass(PassRegistry&);
 void initializeLDTLSCleanupPass(PassRegistry&);
 } // end namespace llvm
 
 #endif

lib/Target/AArch64/AArch64CallLowering.cpp

bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
                                     const MachineOperand &Callee,
                                     const ArgInfo &OrigRet,
                                     ArrayRef<ArgInfo> OrigArgs) const {
   MachineFunction &MF = MIRBuilder.getMF();
   const Function &F = *MF.getFunction();
   MachineRegisterInfo &MRI = MF.getRegInfo();
   auto &DL = F.getParent()->getDataLayout();
 
+  // Can't handle PIP
+  const Function *CalleeF = Callee.isGlobal() ? dyn_cast<Function>(Callee.getGlobal()) : nullptr;
+  if (F.isPagerando() || (CalleeF && CalleeF->isPagerando()))
+    return false;
+
   SmallVector<ArgInfo, 8> SplitArgs;
   for (auto &OrigArg : OrigArgs) {
     splitToValueTypes(OrigArg, SplitArgs, DL, MRI, CallConv,
                       [&](unsigned Reg, uint64_t Offset) {
                         MIRBuilder.buildExtract(Reg, OrigArg.Reg, Offset);
                       });
   }
 

lib/Target/AArch64/AArch64CollectLOH.cpp

static bool canDefBePartOfLOH(const MachineInstr &MI) {
   case AArch64::ADDXri:
     return canAddBePartOfLOH(MI);
   case AArch64::LDRXui:
     // Check immediate to see if the immediate is an address.
     switch (MI.getOperand(2).getType()) {
     default:
       return false;
     case MachineOperand::MO_GlobalAddress:
-      return MI.getOperand(2).getTargetFlags() & AArch64II::MO_GOT;
+      return (MI.getOperand(2).getTargetFlags() & AArch64II::MO_SOURCE) == AArch64II::MO_GOT;
     }
   }
 }
 
 /// Check whether the given instruction can the end of a LOH chain involving a
 /// store.
 static bool isCandidateStore(const MachineInstr &MI, const MachineOperand &MO) {
   switch (MI.getOpcode()) {
static bool isCandidateLoad(const MachineInstr &MI) {
   case AArch64::LDRSWui:
   case AArch64::LDRBui:
   case AArch64::LDRHui:
   case AArch64::LDRWui:
   case AArch64::LDRXui:
   case AArch64::LDRSui:
   case AArch64::LDRDui:
   case AArch64::LDRQui:
-    return !(MI.getOperand(2).getTargetFlags() & AArch64II::MO_GOT);
+    return (MI.getOperand(2).getTargetFlags() & AArch64II::MO_SOURCE) != AArch64II::MO_GOT;
   }
 }
 
 /// Check whether the given instruction can load a litteral.
 static bool supportLoadFromLiteral(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   default:
     return false;
if (isCandidateLoad(MI)) {
     Info.IsCandidate = true;
     Info.MI0 = &MI;
     Info.MI1 = nullptr;
   } else if (MI.getOpcode() == AArch64::ADDXri) {
     Info.Type = MCLOH_AdrpAdd;
     Info.IsCandidate = true;
     Info.MI0 = &MI;
   } else if (MI.getOpcode() == AArch64::LDRXui &&
-             MI.getOperand(2).getTargetFlags() & AArch64II::MO_GOT) {
+             (MI.getOperand(2).getTargetFlags() & AArch64II::MO_SOURCE) == AArch64II::MO_GOT) {
     Info.Type = MCLOH_AdrpLdrGot;
     Info.IsCandidate = true;
     Info.MI0 = &MI;
   }
 }
 
 /// Update state \p Info given the tracked register is clobbered.
 static void handleClobber(LOHInfo &Info) {

lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp

if (MO1.isGlobal()) {
                                     AArch64II::MO_NC);
     }
 
     transferImpOps(MI, MIB1, MIB2);
     MI.eraseFromParent();
     return true;
   }
 
+  case AArch64::LOADgotr: {
+    unsigned DstReg = MI.getOperand(0).getReg();
+    const MachineOperand &Base = MI.getOperand(1);
+    const MachineOperand &Global = MI.getOperand(2);
+    unsigned Flags = Global.getTargetFlags();
+
+    if (Global.isGlobal()) {
+      MachineInstrBuilder MIB =
+          BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::LDRXui), DstReg)
+              .add(Base)
+              .addGlobalAddress(Global.getGlobal(), 0, Flags);
+      transferImpOps(MI, MIB, MIB);
+    } else if (Global.isSymbol()) {
+      MachineInstrBuilder MIB =
+          BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::LDRXui), DstReg)
+              .add(Base)
+              .addExternalSymbol(Global.getSymbolName(), Flags);
+      transferImpOps(MI, MIB, MIB);
+    } else {
+      assert(Global.isReg() &&
+             "Only expect global immediate or register offset");
+      MachineInstrBuilder MIB =
+          BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::LDRXroX), DstReg)
+              .add(Base)
+              .add(Global)
+              .addImm(0)
+              .addImm(0);
+      transferImpOps(MI, MIB, MIB);
+    }
+    MI.eraseFromParent();
+    return true;
+  }
+
+  case AArch64::LOADpot: {
+    unsigned DstReg = MI.getOperand(0).getReg();
+    const MachineOperand &Base = MI.getOperand(1);
+    const MachineOperand &Offset = MI.getOperand(2);
+    unsigned Flags = Offset.getTargetFlags();
+
+    MachineInstrBuilder MIB =
+        BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::LDRXui), DstReg)
+            .add(Base);
+
+    if (Offset.isGlobal()) {
+      MIB.addGlobalAddress(Offset.getGlobal(), 0, Flags | AArch64II::MO_POT);
+    } else if (Offset.isImm()) {
+      MIB.addImm(Offset.getImm());
+    } else {
+      assert(Offset.isCPI() && "Only expect globals, immediates, or constant pools");
+      MIB.addConstantPoolIndex(Offset.getIndex(), Offset.getOffset(),
+                               Flags | AArch64II::MO_POT);
+
+    }
+
+    transferImpOps(MI, MIB, MIB);
+    MI.eraseFromParent();
+    return true;
+  }
+
+  case AArch64::MOVaddrBIN: {
+    unsigned DstReg = MI.getOperand(0).getReg();
+    const MachineOperand &Base = MI.getOperand(1);
+    const MachineOperand &Global = MI.getOperand(2);
+    unsigned Flags = Global.getTargetFlags();
+
+    // TODO(sjc): We need to add a page index to the bin address because we
+    // don't (yet) enforce that bins are <= 4096 bytes. If we can ensure that at
+    // least all destinations in a bin are on the first page, we can drop this
+    // instruction.
+    MachineInstrBuilder MIB1 =
+        BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri), DstReg)
+            .add(Base)
+            .addGlobalAddress(Global.getGlobal(), 0, Flags | AArch64II::MO_HI12)
+            .addImm(12);
+
+    MachineInstrBuilder MIB2 =
+        BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri), DstReg)
+            .addReg(DstReg)
+            .addGlobalAddress(Global.getGlobal(), 0, Flags | AArch64II::MO_PAGEOFF)
+            .addImm(0);
+
+    transferImpOps(MI, MIB1, MIB2);
+    MI.eraseFromParent();
+    return true;
+  }
+
   case AArch64::MOVaddr:
   case AArch64::MOVaddrJT:
   case AArch64::MOVaddrCP:
   case AArch64::MOVaddrBA:
   case AArch64::MOVaddrTLS:
   case AArch64::MOVaddrEXT: {
     // Expand into ADRP + ADD.
     unsigned DstReg = MI.getOperand(0).getReg();

lib/Target/AArch64/AArch64FastISel.cpp

unsigned AArch64FastISel::materializeFP(const ConstantFP *CFP, MVT VT) {
   return ResultReg;
 }
 
 unsigned AArch64FastISel::materializeGV(const GlobalValue *GV) {
   // We can't handle thread-local variables quickly yet.
   if (GV->isThreadLocal())
     return 0;
 
+  auto F = dyn_cast<Function>(GV);
+  if (MF->getFunction()->isPagerando() ||
+      (F && F->isPagerando())) {
+    // TODO(sjc): Implement PIP
+    return 0;
+  }
+
   // MachO still uses GOT for large code-model accesses, but ELF requires
   // movz/movk sequences, which FastISel doesn't handle yet.
   if (!Subtarget->useSmallAddressing() && !Subtarget->isTargetMachO())
     return 0;
 
   unsigned char OpFlags = Subtarget->ClassifyGlobalReference(GV, TM);
 
   EVT DestEVT = TLI.getValueType(DL, GV->getType(), true);
   if (!DestEVT.isSimple())
     return 0;
 
   unsigned ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
   unsigned ResultReg;
 
-  if (OpFlags & AArch64II::MO_GOT) {
+  if (OpFlags == AArch64II::MO_GOT) {
     // ADRP + LDRX
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::ADRP),
             ADRPReg)
       .addGlobalAddress(GV, 0, AArch64II::MO_GOT | AArch64II::MO_PAGE);
 
     ResultReg = createResultReg(&AArch64::GPR64RegClass);
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::LDRXui),
             ResultReg)
bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
   // FIXME: Add large code model support for ELF.
   if (CM == CodeModel::Large && !Subtarget->isTargetMachO())
     return false;
 
   // Let SDISel handle vararg functions.
   if (IsVarArg)
     return false;
 
+  // Can't handle PIP
+  const Function *F = dyn_cast<Function>(Callee);
+  if (FuncInfo.MF->getFunction()->isPagerando() || (F && F->isPagerando()))
+    return false;
+
   // FIXME: Only handle *simple* calls for now.
   MVT RetVT;
   if (CLI.RetTy->isVoidTy())
     RetVT = MVT::isVoid;
   else if (!isTypeLegal(CLI.RetTy, RetVT))
     return false;
 
   for (auto Flag : CLI.OutFlags)

lib/Target/AArch64/AArch64ISelLowering.h

enum NodeType : unsigned {
 
   // Produces the full sequence of instructions for getting the thread pointer
   // offset of a variable into X0, using the TLSDesc model.
   TLSDESC_CALLSEQ,
   ADRP,     // Page address of a TargetGlobalAddress operand.
   ADDlow,   // Add the low 12 bits of a TargetGlobalAddress operand.
   LOADgot,  // Load from automatically generated descriptor (e.g. Global
             // Offset Table, TLS record).
+  LOADgotr,  // Load from automatically generated descriptor (e.g. Global
+             // Offset Table, TLS record) via a register base address.
+  LOADpot, // Load from page linking table (POT)
   RET_FLAG, // Return with a flag operand. Operand 0 is the chain operand.
   BRCOND,   // Conditional branch instruction; "b.cond".
   CSEL,
   FCSEL, // Conditional move instruction.
   CSINV, // Conditional select invert.
   CSNEG, // Conditional select negate.
   CSINC, // Conditional select increment.
 
private:
   SDValue getTargetNode(GlobalAddressSDNode *N, EVT Ty, SelectionDAG &DAG,
                         unsigned Flag) const;
   SDValue getTargetNode(JumpTableSDNode *N, EVT Ty, SelectionDAG &DAG,
                         unsigned Flag) const;
   SDValue getTargetNode(ConstantPoolSDNode *N, EVT Ty, SelectionDAG &DAG,
                         unsigned Flag) const;
   SDValue getTargetNode(BlockAddressSDNode *N, EVT Ty, SelectionDAG &DAG,
                         unsigned Flag) const;
+  SDValue getTargetNode(ExternalSymbolSDNode *N, EVT Ty, SelectionDAG &DAG,
+                        unsigned Flag) const;
   template <class NodeTy> SDValue getGOT(NodeTy *N, SelectionDAG &DAG) const;
+  template <class NodeTy> SDValue getPOT(NodeTy *N, SelectionDAG &DAG) const;
   template <class NodeTy>
   SDValue getAddrLarge(NodeTy *N, SelectionDAG &DAG) const;
   template <class NodeTy> SDValue getAddr(NodeTy *N, SelectionDAG &DAG) const;
   SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerDarwinGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerELFGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerELFTLSDescCallSeq(SDValue SymAddr, const SDLoc &DL,
                                  SelectionDAG &DAG) const;
   SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, SDValue RHS,
                          SDValue TVal, SDValue FVal, const SDLoc &dl,
                          SelectionDAG &DAG) const;
+  SDValue LowerPOT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerAAPCS_VASTART(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerDarwin_VASTART(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerWin64_VASTART(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const;

lib/Target/AArch64/AArch64ISelLowering.cpp

AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   // When comparing vectors the result sets the different elements in the
   // vector to all-one or all-zero.
   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
   // Set up the register classes.
   addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass);
   addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass);
 
+  // TODO: Decide if we want to stick with the platform register
+  if (TM.isPagerando())
+    setPOTBaseRegister(AArch64::X20);
+
   if (Subtarget->hasFPARMv8()) {
     addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
     addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
     addRegisterClass(MVT::f64, &AArch64::FPR64RegClass);
     addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
   }
 
   if (Subtarget->hasNEON()) {
AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand);
 
   // Constant pool entries
   setOperationAction(ISD::ConstantPool, MVT::i64, Custom);
 
   // BlockAddress
   setOperationAction(ISD::BlockAddress, MVT::i64, Custom);
 
+  setOperationAction(ISD::PAGE_OFFSET_TABLE, MVT::i64, Custom);
+
   // Add/Sub overflow ops with MVT::Glues are lowered to NZCV dependences.
   setOperationAction(ISD::ADDC, MVT::i32, Custom);
   setOperationAction(ISD::ADDE, MVT::i32, Custom);
   setOperationAction(ISD::SUBC, MVT::i32, Custom);
   setOperationAction(ISD::SUBE, MVT::i32, Custom);
   setOperationAction(ISD::ADDC, MVT::i64, Custom);
   setOperationAction(ISD::ADDE, MVT::i64, Custom);
   setOperationAction(ISD::SUBC, MVT::i64, Custom);

 
 const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
   switch ((AArch64ISD::NodeType)Opcode) {
   case AArch64ISD::FIRST_NUMBER:      break;
   case AArch64ISD::CALL:              return "AArch64ISD::CALL";
   case AArch64ISD::ADRP:              return "AArch64ISD::ADRP";
   case AArch64ISD::ADDlow:            return "AArch64ISD::ADDlow";
   case AArch64ISD::LOADgot:           return "AArch64ISD::LOADgot";
+  case AArch64ISD::LOADgotr:          return "AArch64ISD::LOADgotr";
+  case AArch64ISD::LOADpot:           return "AArch64ISD::LOADpot";
   case AArch64ISD::RET_FLAG:          return "AArch64ISD::RET_FLAG";
   case AArch64ISD::BRCOND:            return "AArch64ISD::BRCOND";
   case AArch64ISD::CSEL:              return "AArch64ISD::CSEL";
   case AArch64ISD::FCSEL:             return "AArch64ISD::FCSEL";
   case AArch64ISD::CSINV:             return "AArch64ISD::CSINV";
   case AArch64ISD::CSNEG:             return "AArch64ISD::CSNEG";
   case AArch64ISD::CSINC:             return "AArch64ISD::CSINC";
   case AArch64ISD::THREAD_POINTER:    return "AArch64ISD::THREAD_POINTER";
SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
   case ISD::BITCAST:
     return LowerBITCAST(Op, DAG);
   case ISD::GlobalAddress:
     return LowerGlobalAddress(Op, DAG);
   case ISD::GlobalTLSAddress:
     return LowerGlobalTLSAddress(Op, DAG);
   case ISD::SETCC:
     return LowerSETCC(Op, DAG);
+  case ISD::PAGE_OFFSET_TABLE:
+    return LowerPOT(Op, DAG);
   case ISD::BR_CC:
     return LowerBR_CC(Op, DAG);
   case ISD::SELECT:
     return LowerSELECT(Op, DAG);
   case ISD::SELECT_CC:
     return LowerSELECT_CC(Op, DAG);
   case ISD::JumpTable:
     return LowerJumpTable(Op, DAG);
Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first,
                              RegToPass.second, InFlag);
     InFlag = Chain.getValue(1);
   }
 
   // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
   // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
   // node so that legalize doesn't hack it.
   if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
-    auto GV = G->getGlobal();
-    if (Subtarget->classifyGlobalFunctionReference(GV, getTargetMachine()) ==
-        AArch64II::MO_GOT) {
+    auto *GV = G->getGlobal();
+    auto *F = dyn_cast<Function>(GV);
+    if (auto *GA = dyn_cast<GlobalAlias>(GV)) {
+      if (auto *Aliasee = dyn_cast<GlobalValue>(GA->getAliasee()))
+        F = dyn_cast<Function>(Aliasee);
+    }
+    bool UsePIPAddressing = MF.getFunction()->isPagerando() ||
+                            (F && F->isPagerando());
+    unsigned char OpFlags =
+      Subtarget->classifyGlobalFunctionReference(GV, getTargetMachine());
+    if (UsePIPAddressing) {
+      Callee = getPOT(G, DAG);
+    } else if (OpFlags == AArch64II::MO_GOT) {
       Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_GOT);
       Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
     } else {
       const GlobalValue *GV = G->getGlobal();
       Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 0);
     }
   } else if (auto *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
-    if (getTargetMachine().getCodeModel() == CodeModel::Large &&
+    if (MF.getFunction()->isPagerando()) {
+      // Calls from PIP functions should go through the GOT for now so that we
+      // can properly indirect through POT.
+      Callee = getPOT(S, DAG);
+    } else if (getTargetMachine().getCodeModel() == CodeModel::Large &&
         Subtarget->isTargetMachO()) {
       const char *Sym = S->getSymbol();
       Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, AArch64II::MO_GOT);
       Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
     } else {
       const char *Sym = S->getSymbol();
       Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, 0);
     }

 }
 
 SDValue AArch64TargetLowering::getTargetNode(BlockAddressSDNode* N, EVT Ty,
                                              SelectionDAG &DAG,
                                              unsigned Flag) const {
   return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, 0, Flag);
 }
 
+SDValue AArch64TargetLowering::getTargetNode(ExternalSymbolSDNode* N, EVT Ty,
+                                             SelectionDAG &DAG,
+                                             unsigned Flag) const {
+  return DAG.getTargetExternalSymbol(N->getSymbol(), Ty, Flag);
+}
+
 // (loadGOT sym)
 template <class NodeTy>
 SDValue AArch64TargetLowering::getGOT(NodeTy *N, SelectionDAG &DAG) const {
   DEBUG(dbgs() << "AArch64TargetLowering::getGOT\n");
   SDLoc DL(N);
   EVT Ty = getPointerTy(DAG.getDataLayout());
   SDValue GotAddr = getTargetNode(N, Ty, DAG, AArch64II::MO_GOT);
   // FIXME: Once remat is capable of dealing with instructions with register
   // operands, expand this into two nodes instead of using a wrapper node.
   return DAG.getNode(AArch64ISD::LOADgot, DL, Ty, GotAddr);
 }
 
+// Position-independent pages, access through the POT
+template <class NodeTy>
+SDValue AArch64TargetLowering::getPOT(NodeTy *N, SelectionDAG &DAG) const {
+  DEBUG(dbgs() << "AArch64TargetLowering::getPOT\n");
+
+  // Pagerando targets DSOs specifically, and the large code model can only be
+  // used for statically linked binaries. Thus, we do not support a large code
+  // model.
+  assert(getTargetMachine().getCodeModel() != CodeModel::Large);
+
+  SDLoc DL(N);
+  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  const Function *F = nullptr;
+  unsigned char OpFlags = 0;
+  if (auto *GN = dyn_cast<GlobalAddressSDNode>(N)) {
+    const GlobalValue *GV = GN->getGlobal();
+    F = dyn_cast<Function>(GV);
+    OpFlags = Subtarget->ClassifyGlobalReference(GV, getTargetMachine());
+    if (auto *GA = dyn_cast<GlobalAlias>(GV)) {
+      if (auto *Aliasee = dyn_cast<GlobalValue>(GA->getAliasee()))
+        F = dyn_cast<Function>(Aliasee);
+    }
+  }
+  bool pagerandoBinTarget = F && F->isPagerando();
+
+  SDValue POTValue = DAG.getNode(ISD::PAGE_OFFSET_TABLE, DL,
+                                 DAG.getVTList(MVT::i64, MVT::Other));
+  SDValue Chain = POTValue.getValue(1);
+
+  if (OpFlags == AArch64II::MO_GOT) {
+    // Load the GOT address from the POT
+    SDValue GOTAddr = DAG.getNode(AArch64ISD::LOADpot, DL, PtrVT, Chain, POTValue,
+                                  DAG.getTargetConstant(0, DL, MVT::i32));
+
+    const Module *M = DAG.getMachineFunction().getFunction()->getParent();
+    PICLevel::Level picLevel = M->getPICLevel();
+
+    SDValue Offset;
+    const unsigned char MO_NC = AArch64II::MO_NC;
+    const unsigned char MO_GOTOFF = AArch64II::MO_GOTOFF;
+    if (picLevel == PICLevel::SmallPIC) {
+      // GOT size <= 28KiB
+      Offset = getTargetNode(N, PtrVT, DAG, MO_GOTOFF);
+    } else {
+      // Large GOT size
+      Offset = DAG.getNode(
+          AArch64ISD::WrapperLarge, DL, PtrVT,
+          getTargetNode(N, PtrVT, DAG, MO_GOTOFF | AArch64II::MO_G3),
+          getTargetNode(N, PtrVT, DAG, MO_GOTOFF | AArch64II::MO_G2 | MO_NC),
+          getTargetNode(N, PtrVT, DAG, MO_GOTOFF | AArch64II::MO_G1 | MO_NC),
+          getTargetNode(N, PtrVT, DAG, MO_GOTOFF | AArch64II::MO_G0 | MO_NC));
+    }
+
+    return DAG.getNode(AArch64ISD::LOADgotr, DL, PtrVT, GOTAddr, Offset);
+  } else if (pagerandoBinTarget) {
+    SDValue POTOffset = getTargetNode(N, PtrVT, DAG, AArch64II::MO_POT);
+    SDValue BaseAddr = DAG.getNode(AArch64ISD::LOADpot, DL, PtrVT, Chain,
+                                   POTValue, POTOffset);
+
+    SDValue Offset = getTargetNode(N, PtrVT, DAG, AArch64II::MO_SEC);
+
+    return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, Offset);
+  } else {
+    // Load the GOT address from the POT
+    SDValue GOTAddr = DAG.getNode(AArch64ISD::LOADpot, DL, PtrVT, Chain, POTValue,
+                                  DAG.getTargetConstant(0, DL, MVT::i32));
+
+    SDValue Hi = getTargetNode(N, PtrVT, DAG, AArch64II::MO_PAGE);
+    SDValue Lo = getTargetNode(N, PtrVT, DAG,
+                                   AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
+
+    SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, Hi);
+    SDValue TargetPCRel = DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, Lo);
+
+    Hi = DAG.getTargetExternalSymbol("_GLOBAL_OFFSET_TABLE_", PtrVT,
+                                     AArch64II::MO_PAGE);
+    Lo = DAG.getTargetExternalSymbol("_GLOBAL_OFFSET_TABLE_", PtrVT,
+                                     AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
+
+    ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, Hi);
+    SDValue GOTPCRel = DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, Lo);
+
+    SDValue Offset = DAG.getNode(ISD::SUB, DL, PtrVT, TargetPCRel, GOTPCRel);
+
+    return DAG.getNode(ISD::ADD, DL, PtrVT, GOTAddr, Offset);
+  }
+}
+
 // (wrapper %highest(sym), %higher(sym), %hi(sym), %lo(sym))
 template <class NodeTy>
 SDValue AArch64TargetLowering::getAddrLarge(NodeTy *N, SelectionDAG &DAG)
   const {
   DEBUG(dbgs() << "AArch64TargetLowering::getAddrLarge\n");
   SDLoc DL(N);
   EVT Ty = getPointerTy(DAG.getDataLayout());
   const unsigned char MO_NC = AArch64II::MO_NC;
SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
   GlobalAddressSDNode *GN = cast<GlobalAddressSDNode>(Op);
   const GlobalValue *GV = GN->getGlobal();
   unsigned char OpFlags =
       Subtarget->ClassifyGlobalReference(GV, getTargetMachine());
 
   assert(cast<GlobalAddressSDNode>(Op)->getOffset() == 0 &&
          "unexpected offset in global node");
 
+  MachineFunction &MF = DAG.getMachineFunction();
+  auto *F = dyn_cast<Function>(GV);
+  if (auto *GA = dyn_cast<GlobalAlias>(GV)) {
+    if (auto *Aliasee = dyn_cast<GlobalValue>(GA->getAliasee()))
+      F = dyn_cast<Function>(Aliasee);
+  }
+  bool pagerandoBinTarget = F && F->isPagerando();
+  if (MF.getFunction()->isPagerando() ||
+      pagerandoBinTarget) {
+    return getPOT(GN, DAG);
+  }
+
   // This also catches the large code model case for Darwin.
   if ((OpFlags & AArch64II::MO_GOT) != 0) {
     return getGOT(GN, DAG);
   }
 
   if (getTargetMachine().getCodeModel() == CodeModel::Large) {
     return getAddrLarge(GN, DAG);
   } else {
if (CC2 == AArch64CC::AL) {
     SDValue CS1 =
         DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
 
     SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
     return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
   }
 }
 
+SDValue AArch64TargetLowering::LowerPOT(SDValue Op, SelectionDAG &DAG) const {
+  assert(getTargetMachine().isPagerando() &&
+         "POT lowering only supported with PIP relocation model");
+
+  SDLoc dl(Op);
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
+  MachineFunction &MF = DAG.getMachineFunction();
+  unsigned POTReg = getPOTBaseRegister();
+  if (MF.getFunction()->isPagerando()) {
+    return DAG.getCopyFromReg(DAG.getEntryNode(), dl, POTReg, PtrVT);
+  } else {
+    SDValue POTAddress = DAG.getTargetExternalSymbol("_PAGE_OFFSET_TABLE_", PtrVT,
+                                                     AArch64II::MO_GOT);
+    POTAddress = DAG.getNode(AArch64ISD::LOADgot, dl, PtrVT, POTAddress);
+    SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, POTReg, POTAddress);
+    SDValue Ops[2] = { POTAddress, Chain };
+    return DAG.getMergeValues(Ops, dl);
+  }
+}
+
 SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
                                               SDValue RHS, SDValue TVal,
                                               SDValue FVal, const SDLoc &dl,
                                               SelectionDAG &DAG) const {
   // Handle f128 first, because it will result in a comparison of some RTLIB
   // call result against zero.
   if (LHS.getValueType() == MVT::f128) {
     softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl);
SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
   }
   return getAddr(JT, DAG);
 }
 
 SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
                                                  SelectionDAG &DAG) const {
   ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
 
+  if (DAG.getMachineFunction().getFunction()->isPagerando()) {
+    return getPOT(CP, DAG);
+  }
+
   if (getTargetMachine().getCodeModel() == CodeModel::Large) {
     // Use the GOT for the large code model on iOS.
     if (Subtarget->isTargetMachO()) {
       return getGOT(CP, DAG);
     }
     return getAddrLarge(CP, DAG);
   } else {
     return getAddr(CP, DAG);

lib/Target/AArch64/AArch64InstrInfo.cpp

bool AArch64InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
   DebugLoc DL = MI.getDebugLoc();
   unsigned Reg = MI.getOperand(0).getReg();
   const GlobalValue *GV =
       cast<GlobalValue>((*MI.memoperands_begin())->getValue());
   const TargetMachine &TM = MBB.getParent()->getTarget();
   unsigned char OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
   const unsigned char MO_NC = AArch64II::MO_NC;
 
-  if ((OpFlags & AArch64II::MO_GOT) != 0) {
+  if (OpFlags == AArch64II::MO_GOT) {
     BuildMI(MBB, MI, DL, get(AArch64::LOADgot), Reg)
         .addGlobalAddress(GV, 0, AArch64II::MO_GOT);
     BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
         .addReg(Reg, RegState::Kill)
         .addImm(0)
         .addMemOperand(*MI.memoperands_begin());
   } else if (TM.getCodeModel() == CodeModel::Large) {
     BuildMI(MBB, MI, DL, get(AArch64::MOVZXi), Reg)

lib/Target/AArch64/AArch64InstrInfo.td

def SDT_AArch64WrapperLarge : SDTypeProfile<1, 4,
                                          SDTCisSameAs<1, 2>, SDTCisSameAs<1, 3>,
                                          SDTCisSameAs<1, 4>]>;
 
 
 // Node definitions.
 def AArch64adrp          : SDNode<"AArch64ISD::ADRP", SDTIntUnaryOp, []>;
 def AArch64addlow        : SDNode<"AArch64ISD::ADDlow", SDTIntBinOp, []>;
 def AArch64LOADgot       : SDNode<"AArch64ISD::LOADgot", SDTIntUnaryOp>;
+def AArch64LOADgotr      : SDNode<"AArch64ISD::LOADgotr", SDTIntBinOp>;
+def AArch64LOADpot      : SDNode<"AArch64ISD::LOADpot", SDTIntBinOp,
+                                  [SDNPHasChain]>;
 def AArch64callseq_start : SDNode<"ISD::CALLSEQ_START",
                                 SDCallSeqStart<[ SDTCisVT<0, i32>,
                                                  SDTCisVT<1, i32> ]>,
                                 [SDNPHasChain, SDNPOutGlue]>;
 def AArch64callseq_end   : SDNode<"ISD::CALLSEQ_END",
                                 SDCallSeqEnd<[ SDTCisVT<0, i32>,
                                                SDTCisVT<1, i32> ]>,
                                 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;

 // cannot handle multiple instructions.  When that changes, they can be
 // removed, along with the AArch64Wrapper node.
 
 let AddedComplexity = 10 in
 def LOADgot : Pseudo<(outs GPR64:$dst), (ins i64imm:$addr),
                      [(set GPR64:$dst, (AArch64LOADgot tglobaladdr:$addr))]>,
               Sched<[WriteLDAdr]>;
 
+let AddedComplexity = 10 in
+def LOADgotr : Pseudo<(outs GPR64:$dst), (ins GPR64:$base, i64imm:$addr),
+                      [(set GPR64:$dst, (AArch64LOADgotr GPR64:$base, tglobaladdr:$addr))]>,
+               Sched<[WriteLD]>;
+
+def LOADpot : Pseudo<(outs GPR64:$dst), (ins GPR64:$base, i64imm:$addr),
+                      [(set GPR64:$dst, (AArch64LOADpot GPR64:$base, tglobaladdr:$addr))]>,
+               Sched<[WriteLD]>;
+
 // The MOVaddr instruction should match only when the add is not folded
 // into a load or store address.
 def MOVaddr
     : Pseudo<(outs GPR64:$dst), (ins i64imm:$hi, i64imm:$low),
              [(set GPR64:$dst, (AArch64addlow (AArch64adrp tglobaladdr:$hi),
                                             tglobaladdr:$low))]>,
       Sched<[WriteAdrAdr]>;
 def MOVaddrJT
: Pseudo<(outs GPR64:$dst), (ins i64imm:$hi, i64imm:$low),
              [(set GPR64:$dst, (AArch64addlow (AArch64adrp tglobaltlsaddr:$hi),
                                             tglobaltlsaddr:$low))]>,
       Sched<[WriteAdrAdr]>;
 def MOVaddrEXT
     : Pseudo<(outs GPR64:$dst), (ins i64imm:$hi, i64imm:$low),
              [(set GPR64:$dst, (AArch64addlow (AArch64adrp texternalsym:$hi),
                                             texternalsym:$low))]>,
       Sched<[WriteAdrAdr]>;
+def MOVaddrBIN
+    : Pseudo<(outs GPR64:$dst), (ins GPR64:$base, i64imm:$low),
+             [(set GPR64:$dst, (add GPR64:$base, tglobaladdr:$low))]>,
+      Sched<[WriteAdr]>;
 
 } // isReMaterializable, isCodeGenOnly
 
 def : Pat<(AArch64LOADgot tglobaltlsaddr:$addr),
           (LOADgot tglobaltlsaddr:$addr)>;
 
 def : Pat<(AArch64LOADgot texternalsym:$addr),
           (LOADgot texternalsym:$addr)>;
 
 def : Pat<(AArch64LOADgot tconstpool:$addr),
           (LOADgot tconstpool:$addr)>;
 
+def : Pat<(AArch64LOADgotr GPR64:$base, GPR64:$offset),
+          (LOADgotr GPR64:$base, GPR64:$offset)>;
+
+def : Pat<(AArch64LOADpot GPR64:$base, timm:$index),
+          (LOADpot GPR64:$base, imm:$index)>;
+
+def : Pat<(AArch64LOADpot GPR64:$base, tconstpool:$index),
+          (LOADpot GPR64:$base, tconstpool:$index)>;
+
 //===----------------------------------------------------------------------===//
 // System instructions.
 //===----------------------------------------------------------------------===//
 
 def HINT : HintI<"hint">;
 def : InstAlias<"nop",  (HINT 0b000)>;
 def : InstAlias<"yield",(HINT 0b001)>;
 def : InstAlias<"wfe",  (HINT 0b010)>;

 
 def : Pat<(AArch64WrapperLarge tjumptable:$g3, tjumptable:$g2,
                              tjumptable:$g1, tjumptable:$g0),
           (MOVKXi (MOVKXi (MOVKXi (MOVZXi tjumptable:$g0, 0),
                                   tjumptable:$g1, 16),
                           tjumptable:$g2, 32),
                   tjumptable:$g3, 48)>;
 
+def : Pat<(AArch64WrapperLarge texternalsym:$g3, texternalsym:$g2,
+                             texternalsym:$g1, texternalsym:$g0),
+          (MOVKXi (MOVKXi (MOVKXi (MOVZXi texternalsym:$g0, 0),
+                                  texternalsym:$g1, 16),
+                          texternalsym:$g2, 32),
+                  texternalsym:$g3, 48)>;
+
 
 //===----------------------------------------------------------------------===//
 // Arithmetic instructions.
 //===----------------------------------------------------------------------===//
 
 // Add/subtract with carry.
 defm ADC : AddSubCarry<0, "adc", "adcs", AArch64adc, AArch64adc_flag>;
 defm SBC : AddSubCarry<1, "sbc", "sbcs", AArch64sbc, AArch64sbc_flag>;

lib/Target/AArch64/AArch64InstructionSelector.cpp

bool AArch64InstructionSelector::select(MachineInstr &I) const {
   }
 
   case TargetOpcode::G_GLOBAL_VALUE: {
     auto GV = I.getOperand(1).getGlobal();
     if (GV->isThreadLocal()) {
       // FIXME: we don't support TLS yet.
       return false;
     }
+    auto F = dyn_cast<Function>(GV);
+    if (MF.getFunction()->isPagerando() ||
+        (F && F->isPagerando())) {
+      // TODO(sjc): Implement PIP
+      return false;
+    }
     unsigned char OpFlags = STI.ClassifyGlobalReference(GV, TM);
     if (OpFlags & AArch64II::MO_GOT) {
       I.setDesc(TII.get(AArch64::LOADgot));
       I.getOperand(1).setTargetFlags(OpFlags);
     } else {
       I.setDesc(TII.get(AArch64::MOVaddr));
       I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
       MachineInstrBuilder MIB(MF, I);

lib/Target/AArch64/AArch64MCInstLower.cpp

AArch64MCInstLower::GetExternalSymbolSymbol(const MachineOperand &MO) const {
   return Printer.GetExternalSymbolSymbol(MO.getSymbolName());
 }
 
 MCOperand AArch64MCInstLower::lowerSymbolOperandDarwin(const MachineOperand &MO,
                                                        MCSymbol *Sym) const {
   // FIXME: We would like an efficient form for this, so we don't have to do a
   // lot of extra uniquing.
   MCSymbolRefExpr::VariantKind RefKind = MCSymbolRefExpr::VK_None;
-  if ((MO.getTargetFlags() & AArch64II::MO_GOT) != 0) {
+  if ((MO.getTargetFlags() & AArch64II::MO_SOURCE) == AArch64II::MO_GOT) {
     if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_PAGE)
       RefKind = MCSymbolRefExpr::VK_GOTPAGE;
     else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) ==
              AArch64II::MO_PAGEOFF)
       RefKind = MCSymbolRefExpr::VK_GOTPAGEOFF;
     else
       llvm_unreachable("Unexpected target flags with MO_GOT on GV operand");
-  } else if ((MO.getTargetFlags() & AArch64II::MO_TLS) != 0) {
+  } else if ((MO.getTargetFlags() & AArch64II::MO_SOURCE) == AArch64II::MO_TLS) {
     if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_PAGE)
       RefKind = MCSymbolRefExpr::VK_TLVPPAGE;
     else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) ==
              AArch64II::MO_PAGEOFF)
       RefKind = MCSymbolRefExpr::VK_TLVPPAGEOFF;
     else
       llvm_unreachable("Unexpected target flags with MO_TLS on GV operand");
   } else {
     if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_PAGE)
       RefKind = MCSymbolRefExpr::VK_PAGE;
     else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) ==
              AArch64II::MO_PAGEOFF)
       RefKind = MCSymbolRefExpr::VK_PAGEOFF;
   }
   const MCExpr *Expr = MCSymbolRefExpr::create(Sym, RefKind, Ctx);
   if (!MO.isJTI() && MO.getOffset())
     Expr = MCBinaryExpr::createAdd(
         Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
   return MCOperand::createExpr(Expr);
 }
 
 MCOperand AArch64MCInstLower::lowerSymbolOperandELF(const MachineOperand &MO,
                                                     MCSymbol *Sym) const {
   uint32_t RefFlags = 0;
+  unsigned SourceFlag = MO.getTargetFlags() & AArch64II::MO_SOURCE;
+  unsigned FragmentFlag = MO.getTargetFlags() & AArch64II::MO_FRAGMENT;
 
-  if (MO.getTargetFlags() & AArch64II::MO_GOT)
+  if (SourceFlag == AArch64II::MO_GOT)
     RefFlags |= AArch64MCExpr::VK_GOT;
-  else if (MO.getTargetFlags() & AArch64II::MO_TLS) {
+  else if (SourceFlag == AArch64II::MO_GOTOFF)
+    RefFlags |= AArch64MCExpr::VK_GOTOFF;
+  else if (SourceFlag == AArch64II::MO_TLS) {
     TLSModel::Model Model;
     if (MO.isGlobal()) {
       const GlobalValue *GV = MO.getGlobal();
       Model = Printer.TM.getTLSModel(GV);
       if (!EnableAArch64ELFLocalDynamicTLSGeneration &&
           Model == TLSModel::LocalDynamic)
         Model = TLSModel::GeneralDynamic;
 
case TLSModel::LocalExec:
       break;
     case TLSModel::LocalDynamic:
       RefFlags |= AArch64MCExpr::VK_DTPREL;
       break;
     case TLSModel::GeneralDynamic:
       RefFlags |= AArch64MCExpr::VK_TLSDESC;
       break;
     }
+  } else if (SourceFlag == AArch64II::MO_SEC) {
+    RefFlags |= AArch64MCExpr::VK_SEC;
   } else {
     // No modifier means this is a generic reference, classified as absolute for
     // the cases where it matters (:abs_g0: etc).
     RefFlags |= AArch64MCExpr::VK_ABS;
   }
 
-  if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_PAGE)
+  if (FragmentFlag == AArch64II::MO_PAGE)
     RefFlags |= AArch64MCExpr::VK_PAGE;
-  else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) ==
+  else if (FragmentFlag ==
            AArch64II::MO_PAGEOFF)
     RefFlags |= AArch64MCExpr::VK_PAGEOFF;
-  else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G3)
+  else if (FragmentFlag == AArch64II::MO_G3)
     RefFlags |= AArch64MCExpr::VK_G3;
-  else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G2)
+  else if (FragmentFlag == AArch64II::MO_G2)
     RefFlags |= AArch64MCExpr::VK_G2;
-  else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G1)
+  else if (FragmentFlag == AArch64II::MO_G1)
     RefFlags |= AArch64MCExpr::VK_G1;
-  else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G0)
+  else if (FragmentFlag == AArch64II::MO_G0)
     RefFlags |= AArch64MCExpr::VK_G0;
-  else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_HI12)
+  else if (FragmentFlag == AArch64II::MO_HI12)
     RefFlags |= AArch64MCExpr::VK_HI12;
 
   if (MO.getTargetFlags() & AArch64II::MO_NC)
     RefFlags |= AArch64MCExpr::VK_NC;
 
   const MCExpr *Expr =
       MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, Ctx);
   if (!MO.isJTI() && MO.getOffset())
     Expr = MCBinaryExpr::createAdd(
         Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
 
+  if (SourceFlag == AArch64II::MO_POT) {
+    unsigned index;
+    if (MO.isGlobal()) {
+      auto *GO = cast<GlobalObject>(MO.getGlobal());
+      index = Printer.GetPOTIndex(GO);
+    } else {
+      assert(MO.isCPI() && "Can only handle globals or constant pool indices");
+      index = Printer.GetPOTIndex(MO.getIndex());
+    }
+    return MCOperand::createImm(index);
+  } else if (SourceFlag == AArch64II::MO_SEC) {
+    const MCSymbol *SecSym;
+    if (MO.isGlobal()) {
+      auto *GO = cast<GlobalObject>(MO.getGlobal());
+      SecSym = Printer.GetSectionSymbol(GO);
+    } else {
+      assert(MO.isCPI() && "Can only handle globals or constant pool indices");
+      SecSym = Printer.GetSectionSymbol(MO.getIndex());
+    }
+    assert(SecSym && "Could not find a section symbol");
+    const MCExpr *SecExpr = MCSymbolRefExpr::create(SecSym, Ctx);
+    Expr = MCBinaryExpr::createSub(Expr, SecExpr, Ctx);
+    if (FragmentFlag == AArch64II::MO_PAGEOFF) {
+      const MCExpr *MaskExpr = MCConstantExpr::create(0xfff, Ctx);
+      Expr = MCBinaryExpr::createAnd(Expr, MaskExpr, Ctx);
+    } else if (FragmentFlag == AArch64II::MO_HI12) {
+      const MCExpr *ShiftExpr = MCConstantExpr::create(12, Ctx);
+      Expr = MCBinaryExpr::createLShr(Expr, ShiftExpr, Ctx);
+    }
+  }
+
   AArch64MCExpr::VariantKind RefKind;
   RefKind = static_cast<AArch64MCExpr::VariantKind>(RefFlags);
   Expr = AArch64MCExpr::create(Expr, RefKind, Ctx);
 
   return MCOperand::createExpr(Expr);
 }
 
 MCOperand AArch64MCInstLower::lowerSymbolOperandCOFF(const MachineOperand &MO,

lib/Target/AArch64/AArch64PagerandoOptimizer.cpp

+//===-- AArch64PagerandoOptimizer.cpp - Optimizes intra-bin function calls ===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass optimizes calls inside the same position-independent bin to direct
+// calls to avoid the overhead of indirect calls through the POT.
+// The implementation relies on SSA form to follow def-use chains, therefore,
+// this pass must be scheduled before register allocation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AArch64.h"
+#include "AArch64MachineFunctionInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetLowering.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "pagerando"
+
+namespace {
+class AArch64PagerandoOptimizer : public MachineFunctionPass {
+public:
+  static char ID;
+  explicit AArch64PagerandoOptimizer() : MachineFunctionPass(ID) {
+    initializeAArch64PagerandoOptimizerPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  MachineFunctionProperties getRequiredProperties() const override {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::TracksLiveness);
+  }
+
+private:
+  void optimizeCalls(MachineInstr *MI);
+  void replaceWithDirectCall(MachineInstr *MI, const Function *Callee);
+};
+} // end anonymous namespace
+
+char AArch64PagerandoOptimizer::ID = 0;
+INITIALIZE_PASS(AArch64PagerandoOptimizer, "pagerando-optimizer-aarch64",
+                "Pagerando intra-bin optimizer for AArch64", false, false)
+
+FunctionPass *llvm::createAArch64PagerandoOptimizerPass() {
+  return new AArch64PagerandoOptimizer();
+}
+
+static const Function *getCallee(const MachineInstr &MI) {
+  assert(MI.getOpcode() == AArch64::MOVaddrBIN);
+  return cast<Function>(MI.getOperand(2).getGlobal());
+}
+
+static bool isIntraBin(const MachineInstr &MI, StringRef BinPrefix) {
+  return MI.getOpcode() == AArch64::MOVaddrBIN
+      && getCallee(MI)->getSectionPrefix() == BinPrefix;
+}
+
+bool AArch64PagerandoOptimizer::runOnMachineFunction(MachineFunction &MF) {
+  auto &F = *MF.getFunction();
+  // This pass is an optimization (optional), therefore check skipFunction
+  if (!F.isPagerando() || skipFunction(F)) {
+    return false;
+  }
+
+  // Section prefix is assigned by PagerandoBinning pass
+  auto BinPrefix = F.getSectionPrefix().getValue();
+
+  // Collect intra-bin references
+  std::vector<MachineInstr*> Worklist;
+  for (auto &BB : MF) {
+    for (auto &MI : BB) {
+      if (isIntraBin(MI, BinPrefix))
+        Worklist.push_back(&MI);
+    }
+  }
+
+  // Optimize intra-bin calls
+  for (auto *MI : Worklist)
+    optimizeCalls(MI);
+
+  return !Worklist.empty();
+}
+
+void AArch64PagerandoOptimizer::optimizeCalls(MachineInstr *MI) {
+  auto &MRI = MI->getParent()->getParent()->getRegInfo();
+
+  SmallVector<MachineInstr*, 2> Calls;
+  for (auto &Op : MI->defs()) {
+    for (auto &User : MRI.use_instructions(Op.getReg()))
+      Calls.push_back(&User);
+  }
+
+  auto *Callee = getCallee(*MI);
+  for (auto *Call : Calls)
+    replaceWithDirectCall(Call, Callee);
+
+  MI->eraseFromParent();
+  // Note: this might be the only use of the preceding AArch64::LOADpot pseudo
+  // instruction. We schedule the DeadMachineInstructionElim pass after this
+  // pass to get rid of it.
+}
+
+static unsigned toDirectCall(unsigned Opc) {
+  switch (Opc) {
+  case AArch64::BLR:        return AArch64::BL;
+  case AArch64::TCRETURNri: return AArch64::TCRETURNdi;
+  default:
+    llvm_unreachable("Unhandled AArch64 call opcode");
+  }
+}
+
+void AArch64PagerandoOptimizer::replaceWithDirectCall(MachineInstr *MI,
+                                                      const Function *Callee) {
+  auto &MBB = *MI->getParent();
+  auto &TII = *MBB.getParent()->getSubtarget().getInstrInfo();
+
+  auto Opc = toDirectCall(MI->getOpcode());
+  auto MIB = BuildMI(MBB, *MI, MI->getDebugLoc(), TII.get(Opc))
+      .addGlobalAddress(Callee);
+
+  // Copy over remaining operands
+  auto RemainingOps = make_range(MI->operands_begin() + 1, MI->operands_end());
+  for (auto &Op : RemainingOps)
+    MIB.add(Op);
+
+  MI->eraseFromParent();
+}

lib/Target/AArch64/AArch64RegisterInfo.cpp

if (TFI->hasFP(MF) || TT.isOSDarwin())
     markSuperRegs(Reserved, AArch64::W29);
 
   if (MF.getSubtarget<AArch64Subtarget>().isX18Reserved())
     markSuperRegs(Reserved, AArch64::W18); // Platform register
 
   if (hasBasePointer(MF))
     markSuperRegs(Reserved, AArch64::W19);
 
+  if (MF.getTarget().isPagerando())
+    markSuperRegs(Reserved, AArch64::W20); // POT register
+
   assert(checkAllSuperRegsMarked(Reserved));
   return Reserved;
 }
 
 bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF,
                                       unsigned Reg) const {
   const AArch64FrameLowering *TFI = getFrameLowering(MF);
 
bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF,
   case AArch64::W18:
     return MF.getSubtarget<AArch64Subtarget>().isX18Reserved();
   case AArch64::FP:
   case AArch64::W29:
     return TFI->hasFP(MF) || TT.isOSDarwin();
   case AArch64::W19:
   case AArch64::X19:
     return hasBasePointer(MF);
+  case AArch64::W20:
+  case AArch64::X20:
+    return MF.getTarget().isPagerando();
   }
 
   return false;
 }
 
 bool AArch64RegisterInfo::isConstantPhysReg(unsigned PhysReg) const {
   return PhysReg == AArch64::WZR || PhysReg == AArch64::XZR;
 }
unsigned AArch64RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
   case AArch64::GPR64allRegClassID:
   case AArch64::GPR64RegClassID:
   case AArch64::GPR32commonRegClassID:
   case AArch64::GPR64commonRegClassID:
     return 32 - 1                                   // XZR/SP
               - (TFI->hasFP(MF) || TT.isOSDarwin()) // FP
               - MF.getSubtarget<AArch64Subtarget>()
                     .isX18Reserved() // X18 reserved as platform register
-              - hasBasePointer(MF);  // X19
+              - hasBasePointer(MF)   // X19
+              - MF.getTarget()
+                    .isPagerando();  // X20 reserved as POT register
   case AArch64::FPR8RegClassID:
   case AArch64::FPR16RegClassID:
   case AArch64::FPR32RegClassID:
   case AArch64::FPR64RegClassID:
   case AArch64::FPR128RegClassID:
     return 32;
 
   case AArch64::DDRegClassID:

lib/Target/AArch64/AArch64TargetMachine.cpp

extern "C" void LLVMInitializeAArch64Target() {
   initializeAArch64ExpandPseudoPass(*PR);
   initializeAArch64LoadStoreOptPass(*PR);
   initializeAArch64VectorByElementOptPass(*PR);
   initializeAArch64PromoteConstantPass(*PR);
   initializeAArch64RedundantCopyEliminationPass(*PR);
   initializeAArch64StorePairSuppressPass(*PR);
   initializeFalkorHWPFFixPass(*PR);
   initializeFalkorMarkStridedAccessesLegacyPass(*PR);
+  initializeAArch64PagerandoOptimizerPass(*PR);
   initializeLDTLSCleanupPass(*PR);
 }
 
 //===----------------------------------------------------------------------===//
 // AArch64 Lowering public interface.
 //===----------------------------------------------------------------------===//
 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
   if (TT.isOSBinFormatMachO())
if (EnableEarlyIfConversion)
     addPass(&EarlyIfConverterID);
   if (EnableStPairSuppress)
     addPass(createAArch64StorePairSuppressPass());
   addPass(createAArch64VectorByElementOptPass());
   return true;
 }
 
 void AArch64PassConfig::addPreRegAlloc() {
+  if (TM->getOptLevel() != CodeGenOpt::None && TM->isPagerando()) {
+    addPass(createAArch64PagerandoOptimizerPass());
+    addPass(&DeadMachineInstructionElimID);
+  }
+
   // Change dead register definitions to refer to the zero register.
   if (TM->getOptLevel() != CodeGenOpt::None && EnableDeadRegisterElimination)
     addPass(createAArch64DeadRegisterDefinitions());
 
   // Use AdvSIMD scalar instructions whenever profitable.
   if (TM->getOptLevel() != CodeGenOpt::None && EnableAdvSIMDScalar) {
     addPass(createAArch64AdvSIMDScalar());
     // The AdvSIMD pass may produce copies that can be rewritten to

lib/Target/AArch64/CMakeLists.txt

add_llvm_target(AArch64CodeGen
   AArch64ISelDAGToDAG.cpp
   AArch64ISelLowering.cpp
   AArch64InstrInfo.cpp
   AArch64InstructionSelector.cpp
   AArch64LegalizerInfo.cpp
   AArch64LoadStoreOptimizer.cpp
   AArch64MacroFusion.cpp
   AArch64MCInstLower.cpp
+  AArch64PagerandoOptimizer.cpp
   AArch64PromoteConstant.cpp
   AArch64PBQPRegAlloc.cpp
   AArch64RegisterBankInfo.cpp
   AArch64RegisterInfo.cpp
   AArch64SelectionDAGInfo.cpp
   AArch64StorePairSuppress.cpp
   AArch64Subtarget.cpp
   AArch64TargetMachine.cpp

lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp

 //===-- AArch64AsmBackend.cpp - AArch64 Assembler Backend -----------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "AArch64.h"
 #include "AArch64RegisterInfo.h"
 #include "MCTargetDesc/AArch64FixupKinds.h"
+#include "MCTargetDesc/AArch64MCExpr.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCFixupKindInfo.h"
default:
     llvm_unreachable("Unknown fixup kind!");
 
   case AArch64::fixup_aarch64_tlsdesc_call:
     return 0;
 
   case FK_Data_1:
     return 1;
 
-  case AArch64::fixup_aarch64_movw:
   case FK_Data_2:
   case FK_SecRel_2:
     return 2;
 
   case AArch64::fixup_aarch64_pcrel_branch14:
   case AArch64::fixup_aarch64_add_imm12:
   case AArch64::fixup_aarch64_ldst_imm12_scale1:
   case AArch64::fixup_aarch64_ldst_imm12_scale2:
   case AArch64::fixup_aarch64_ldst_imm12_scale4:
   case AArch64::fixup_aarch64_ldst_imm12_scale8:
   case AArch64::fixup_aarch64_ldst_imm12_scale16:
   case AArch64::fixup_aarch64_ldr_pcrel_imm19:
   case AArch64::fixup_aarch64_pcrel_branch19:
+  case AArch64::fixup_aarch64_movw:
     return 3;
 
   case AArch64::fixup_aarch64_pcrel_adr_imm21:
   case AArch64::fixup_aarch64_pcrel_adrp_imm21:
   case AArch64::fixup_aarch64_pcrel_branch26:
   case AArch64::fixup_aarch64_pcrel_call26:
   case FK_Data_4:
   case FK_SecRel_4:
case AArch64::fixup_aarch64_ldst_imm12_scale16:
     if (TheTriple.isOSBinFormatCOFF() && !IsResolved)
       Value &= 0xfff;
     // Unsigned 12-bit immediate which gets multiplied by 16
     if (Value >= 0x10000)
       Ctx.reportError(Fixup.getLoc(), "fixup value out of range");
     if (Value & 0xf)
       Ctx.reportError(Fixup.getLoc(), "fixup must be 16-byte aligned");
     return Value >> 4;
-  case AArch64::fixup_aarch64_movw:
+  case AArch64::fixup_aarch64_movw: {
+    const auto *A64E = cast<AArch64MCExpr>(Fixup.getValue());
+    AArch64MCExpr::VariantKind AddressFrag = AArch64MCExpr::getAddressFrag(A64E->getKind());
+    if (!AArch64MCExpr::isNotChecked(A64E->getKind())) {
+      if (AddressFrag == AArch64MCExpr::VK_G0 &&
+          Value >= 0x10000ULL)
+        Ctx.reportError(Fixup.getLoc(), "fixup value out of range");
+      else if (AddressFrag == AArch64MCExpr::VK_G1 &&
+               Value >= 0x100000000ULL)
+        Ctx.reportError(Fixup.getLoc(), "fixup value out of range");
+      else if (AddressFrag == AArch64MCExpr::VK_G2 &&
+               Value >= 0x1000000000000ULL)
+        Ctx.reportError(Fixup.getLoc(), "fixup value out of range");
+    }
+    if (AddressFrag == AArch64MCExpr::VK_G0)
+      return Value & 0xffff;
+    else if (AddressFrag == AArch64MCExpr::VK_G1)
+      return (Value >> 16) & 0xffff;
+    else if (AddressFrag == AArch64MCExpr::VK_G2)
+      return (Value >> 32) & 0xffff;
+    else if (AddressFrag == AArch64MCExpr::VK_G3)
+      return (Value >> 48) & 0xffff;
+
     Ctx.reportError(Fixup.getLoc(),
-                    "no resolvable MOVZ/MOVK fixups supported yet");
+                    "MOVZ/MOVK fixup not recognized");
     return Value;
+  }
   case AArch64::fixup_aarch64_pcrel_branch14:
     // Signed 16-bit immediate
     if (SignedValue > 32767 || SignedValue < -32768)
       Ctx.reportError(Fixup.getLoc(), "fixup value out of range");
     // Low two bits are not encoded (4-byte alignment assumed).
     if (Value & 0x3)
       Ctx.reportError(Fixup.getLoc(), "fixup not sufficiently aligned");
     return (Value >> 2) & 0x3fff;

lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp

case AArch64MCExpr::VK_TPREL_G1_NC:
     Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(TLSLE_MOVW_TPREL_G1_NC));
     return true;
   case AArch64MCExpr::VK_GOTTPREL_G1:
     Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(TLSIE_MOVW_GOTTPREL_G1));
     return true;
   case AArch64MCExpr::VK_GOTTPREL_G0_NC:
     Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(TLSIE_MOVW_GOTTPREL_G0_NC));
     return true;
+  case AArch64MCExpr::VK_GOTOFF:
+    Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(LD64_GOTOFF_LO15));
+    return true;
   default:
     return false;
   }
   return false;
 }
 
 unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx,
                                               const MCValue &Target,
case AArch64::fixup_aarch64_ldst_imm12_scale8:
           return ELF::R_AARCH64_LD64_GOT_LO12_NC;
         } else {
           Ctx.reportError(Fixup.getLoc(), "ILP32 64-bit load/store "
                                           "relocation not supported (LP64 eqv: "
                                           "LD64_GOT_LO12_NC)");
           return ELF::R_AARCH64_NONE;
         }
       }
+      if (SymLoc == AArch64MCExpr::VK_GOTOFF && !IsNC)
+        return ELF::R_AARCH64_LD64_GOTOFF_LO15;
       if (SymLoc == AArch64MCExpr::VK_DTPREL && !IsNC)
         return R_CLS(TLSLD_LDST64_DTPREL_LO12);
       if (SymLoc == AArch64MCExpr::VK_DTPREL && IsNC)
         return R_CLS(TLSLD_LDST64_DTPREL_LO12_NC);
       if (SymLoc == AArch64MCExpr::VK_TPREL && !IsNC)
         return R_CLS(TLSLE_LDST64_TPREL_LO12);
       if (SymLoc == AArch64MCExpr::VK_TPREL && IsNC)
         return R_CLS(TLSLE_LDST64_TPREL_LO12_NC);
case AArch64::fixup_aarch64_movw:
       if (RefKind == AArch64MCExpr::VK_TPREL_G0)
         return R_CLS(TLSLE_MOVW_TPREL_G0);
       if (RefKind == AArch64MCExpr::VK_TPREL_G0_NC)
         return R_CLS(TLSLE_MOVW_TPREL_G0_NC);
       if (RefKind == AArch64MCExpr::VK_GOTTPREL_G1)
         return ELF::R_AARCH64_TLSIE_MOVW_GOTTPREL_G1;
       if (RefKind == AArch64MCExpr::VK_GOTTPREL_G0_NC)
         return ELF::R_AARCH64_TLSIE_MOVW_GOTTPREL_G0_NC;
+      if (RefKind == AArch64MCExpr::VK_GOTOFF_G3)
+        return ELF::R_AARCH64_MOVW_GOTOFF_G3;
+      if (RefKind == AArch64MCExpr::VK_GOTOFF_G2_NC)
+        return ELF::R_AARCH64_MOVW_GOTOFF_G2_NC;
+      if (RefKind == AArch64MCExpr::VK_GOTOFF_G2)
+        return ELF::R_AARCH64_MOVW_GOTOFF_G2;
+      if (RefKind == AArch64MCExpr::VK_GOTOFF_G1_NC)
+        return ELF::R_AARCH64_MOVW_GOTOFF_G1_NC;
+      if (RefKind == AArch64MCExpr::VK_GOTOFF_G1)
+        return ELF::R_AARCH64_MOVW_GOTOFF_G1;
+      if (RefKind == AArch64MCExpr::VK_GOTOFF_G0_NC)
+        return ELF::R_AARCH64_MOVW_GOTOFF_G0_NC;
+      if (RefKind == AArch64MCExpr::VK_GOTOFF_G0)
+        return ELF::R_AARCH64_MOVW_GOTOFF_G0;
       Ctx.reportError(Fixup.getLoc(),
                       "invalid fixup for movz/movk instruction");
       return ELF::R_AARCH64_NONE;
     case AArch64::fixup_aarch64_tlsdesc_call:
       return R_CLS(TLSDESC_CALL);
     default:
       Ctx.reportError(Fixup.getLoc(), "Unknown ELF relocation type");
       return ELF::R_AARCH64_NONE;

lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h

enum VariantKind {
     // symbol. E.g. direct, via the GOT, ...
     VK_ABS      = 0x001,
     VK_SABS     = 0x002,
     VK_GOT      = 0x003,
     VK_DTPREL   = 0x004,
     VK_GOTTPREL = 0x005,
     VK_TPREL    = 0x006,
     VK_TLSDESC  = 0x007,
+    VK_GOTOFF   = 0x008,
+    VK_POT     = 0x009,
+    VK_SEC      = 0x00a,
     VK_SymLocBits = 0x00f,
 
     // Variants specifying which part of the final address calculation is
     // used. E.g. the low 12 bits for an ADD/LDR, the middle 16 bits for a
     // MOVZ/MOVK.
     VK_PAGE     = 0x010,
     VK_PAGEOFF  = 0x020,
     VK_HI12     = 0x030,
enum VariantKind {
     VK_TPREL_G1_NC       = VK_TPREL    | VK_G1      | VK_NC,
     VK_TPREL_G0          = VK_TPREL    | VK_G0,
     VK_TPREL_G0_NC       = VK_TPREL    | VK_G0      | VK_NC,
     VK_TPREL_HI12        = VK_TPREL    | VK_HI12,
     VK_TPREL_LO12        = VK_TPREL    | VK_PAGEOFF,
     VK_TPREL_LO12_NC     = VK_TPREL    | VK_PAGEOFF | VK_NC,
     VK_TLSDESC_LO12      = VK_TLSDESC  | VK_PAGEOFF,
     VK_TLSDESC_PAGE      = VK_TLSDESC  | VK_PAGE,
+    VK_GOTOFF_G3         = VK_GOTOFF   | VK_G3,
+    VK_GOTOFF_G2         = VK_GOTOFF   | VK_G2,
+    VK_GOTOFF_G2_NC      = VK_GOTOFF   | VK_G2      | VK_NC,
+    VK_GOTOFF_G1         = VK_GOTOFF   | VK_G1,
+    VK_GOTOFF_G1_NC      = VK_GOTOFF   | VK_G1      | VK_NC,
+    VK_GOTOFF_G0         = VK_GOTOFF   | VK_G0,
+    VK_GOTOFF_G0_NC      = VK_GOTOFF   | VK_G0      | VK_NC,
+    VK_SEC_HI12          = VK_SEC      | VK_HI12,
+    VK_SEC_LO12          = VK_SEC      | VK_PAGEOFF,
+    VK_SEC_LO12_NC       = VK_SEC      | VK_PAGEOFF | VK_NC,
+    VK_SEC_G3            = VK_SEC      | VK_G3,
+    VK_SEC_G2            = VK_SEC      | VK_G2,
+    VK_SEC_G2_NC         = VK_SEC      | VK_G2      | VK_NC,
+    VK_SEC_G1            = VK_SEC      | VK_G1,
+    VK_SEC_G1_NC         = VK_SEC      | VK_G1      | VK_NC,
+    VK_SEC_G0            = VK_SEC      | VK_G0,
+    VK_SEC_G0_NC         = VK_SEC      | VK_G0      | VK_NC,
 
     VK_INVALID  = 0xfff
   };
 
 private:
   const MCExpr *Expr;
   const VariantKind Kind;
 

lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp

StringRef AArch64MCExpr::getVariantKindName() const {
   case VK_GOT_PAGE:            return ":got:";
   case VK_GOT_LO12:            return ":got_lo12:";
   case VK_GOTTPREL_PAGE:       return ":gottprel:";
   case VK_GOTTPREL_LO12_NC:    return ":gottprel_lo12:";
   case VK_GOTTPREL_G1:         return ":gottprel_g1:";
   case VK_GOTTPREL_G0_NC:      return ":gottprel_g0_nc:";
   case VK_TLSDESC:             return "";
   case VK_TLSDESC_PAGE:        return ":tlsdesc:";
+  case VK_GOTOFF:              return ":gotoff:";
+  case VK_GOTOFF_G3:           return ":gotoff_g3:";
+  case VK_GOTOFF_G2:           return ":gotoff_g2:";
+  case VK_GOTOFF_G2_NC:        return ":gotoff_g2_nc:";
+  case VK_GOTOFF_G1:           return ":gotoff_g1:";
+  case VK_GOTOFF_G1_NC:        return ":gotoff_g1_nc:";
+  case VK_GOTOFF_G0:           return ":gotoff_g0:";
+  case VK_GOTOFF_G0_NC:        return ":gotoff_g0_nc:";
+  case VK_SEC_HI12:            return ":sec_hi12:";
+  case VK_SEC_LO12:            return ":sec_lo12:";
+  case VK_SEC:                 return ":sec:";
+  case VK_SEC_G3:              return ":sec_g3:";
+  case VK_SEC_G2:              return ":sec_g2:";
+  case VK_SEC_G2_NC:           return ":sec_g2_nc:";
+  case VK_SEC_G1:              return ":sec_g1:";
+  case VK_SEC_G1_NC:           return ":sec_g1_nc:";
+  case VK_SEC_G0:              return ":sec_g0:";
+  case VK_SEC_G0_NC:           return ":sec_g0_nc:";
   default:
     llvm_unreachable("Invalid ELF symbol kind");
   }
 }
 
 void AArch64MCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
   if (getKind() != VK_NONE)
     OS << getVariantKindName();

lib/Target/AArch64/Utils/AArch64BaseInfo.h

enum TOF {
     /// 0-15 of a 64-bit address, used in a MOVZ or MOVK instruction
     MO_G0 = 6,
 
     /// MO_HI12 - This flag indicates that a symbol operand represents the bits
     /// 13-24 of a 64-bit address, used in a arithmetic immediate-shifted-left-
     /// by-12-bits instruction.
     MO_HI12 = 7,
 
+    MO_SOURCE = 0x70,
+
     /// MO_GOT - This flag indicates that a symbol operand represents the
     /// address of the GOT entry for the symbol, rather than the address of
     /// the symbol itself.
     MO_GOT = 0x10,
 
-    /// MO_NC - Indicates whether the linker is expected to check the symbol
-    /// reference for overflow. For example in an ADRP/ADD pair of relocations
-    /// the ADRP usually does check, but not the ADD.
-    MO_NC = 0x20,
+    MO_GOTOFF = 0x20,
+
+    MO_POT = 0x30,
 
     /// MO_TLS - Indicates that the operand being accessed is some kind of
     /// thread-local symbol. On Darwin, only one type of thread-local access
     /// exists (pre linker-relaxation), but on ELF the TLSModel used for the
     /// referee will affect interpretation.
-    MO_TLS = 0x40
+    MO_TLS = 0x40,
+
+    MO_SEC = 0x50,
+
+    /// MO_NC - Indicates whether the linker is expected to check the symbol
+    /// reference for overflow. For example in an ADRP/ADD pair of relocations
+    /// the ADRP usually does check, but not the ADD.
+    MO_NC = 0x80,
   };
 } // end namespace AArch64II
 
 } // end namespace llvm
 
 #endif

test/CodeGen/AArch64/pagerando-isel.ll

+; RUN: llc < %s -march=aarch64 -relocation-model=pip -o /dev/null 2>&1 \
+; RUN:     -print-before=pagerando-optimizer-aarch64 | FileCheck %s
+
+@global_var = global i32 0
+@internal_var = internal global i32 0
+
+define void @legacy() { ret void }
+define void @wrapper() pagerando_wrapper { ret void }
+define hidden void @binned() pagerando_binned { ret void }
+
+; CHECK-LABEL: # *** IR Dump Before Pagerando intra-bin optimizer for AArch64 ***:
+; CHECK-LABEL: # Machine code for function user: IsSSA, TracksLiveness
+define void @user() pagerando_binned {
+; CHECK-DAG: [[POT:%vreg[0-9]+]]<def> = COPY %X20
+; CHECK-DAG: [[GOT:%vreg[0-9]+]]<def> = LOADpot [[POT]], 0
+
+; CHECK-DAG: %vreg{{[0-9]+}}<def> = MOVZXi <ga:@legacy>[TF=166], 0
+; CHECK-DAG: %vreg{{[0-9]+}}<def,tied1> = MOVKXi %vreg{{[0-9]+}}<tied0>, <ga:@legacy>[TF=165], 16
+; CHECK-DAG: %vreg{{[0-9]+}}<def,tied1> = MOVKXi %vreg{{[0-9]+}}<tied0>, <ga:@legacy>[TF=164], 32
+; CHECK-DAG: [[LEGACY_GOTOFF:%vreg[0-9]+]]<def,tied1> = MOVKXi %vreg{{[0-9]+}}<tied0>, <ga:@legacy>[TF=35], 48
+; CHECK: [[LEGACY:%vreg[0-9]+]]<def> = LOADgotr [[GOT]], [[LEGACY_GOTOFF]]<kill>
+; CHECK: BLR [[LEGACY]]<kill>
+  call void @legacy()
+
+; CHECK: %vreg{{[0-9]+}}<def> = MOVZXi <ga:@wrapper>[TF=166], 0
+; CHECK: %vreg{{[0-9]+}}<def,tied1> = MOVKXi %vreg{{[0-9]+}}<tied0>, <ga:@wrapper>[TF=165], 16
+; CHECK: %vreg{{[0-9]+}}<def,tied1> = MOVKXi %vreg{{[0-9]+}}<tied0>, <ga:@wrapper>[TF=164], 32
+; CHECK: [[WRAPPER_GOTOFF:%vreg[0-9]+]]<def,tied1> = MOVKXi %vreg{{[0-9]+}}<tied0>, <ga:@wrapper>[TF=35], 48
+; CHECK: [[WRAPPER:%vreg[0-9]+]]<def> = LOADgotr [[GOT]], [[WRAPPER_GOTOFF]]<kill>
+; CHECK: BLR [[WRAPPER]]<kill>
+  call void @wrapper()
+
+; CHECK: [[BINNED_BIN:%vreg[0-9]+]]<def> = LOADpot [[POT]], <ga:@binned>[TF=48]
+; CHECK: [[BINNED:%vreg[0-9]+]]<def> = MOVaddrBIN [[BINNED_BIN]]<kill>, <ga:@binned>[TF=80]
+; CHECK: BLR [[BINNED]]<kill>
+  call void @binned()
+
+; CHECK: %vreg{{[0-9]+}}<def> = MOVZXi <ga:@global_var>[TF=166], 0
+; CHECK: %vreg{{[0-9]+}}<def,tied1> = MOVKXi %vreg{{[0-9]+}}<tied0>, <ga:@global_var>[TF=165], 16
+; CHECK: %vreg{{[0-9]+}}<def,tied1> = MOVKXi %vreg{{[0-9]+}}<tied0>, <ga:@global_var>[TF=164], 32
+; CHECK: [[GLOBAL_VAR_GOTOFF:%vreg[0-9]+]]<def,tied1> = MOVKXi %vreg{{[0-9]+}}<tied0>, <ga:@global_var>[TF=35], 48
+; CHECK: [[GLOBAL_VAR_ADDR:%vreg[0-9]+]]<def> = LOADgotr [[GOT]], [[GLOBAL_VAR_GOTOFF]]<kill>
+; CHECK: [[VAL:%vreg[0-9]+]]<def> = LDRWui [[GLOBAL_VAR_ADDR]]<kill>, 0
+  %val = load i32, i32* @global_var
+
+; CHECK: [[GOT_PCREL:%vreg[0-9]+]]<def> = MOVaddrEXT <es:_GLOBAL_OFFSET_TABLE_>[TF=1], <es:_GLOBAL_OFFSET_TABLE_>[TF=130]
+; CHECK: [[INTERNAL_VAR_PCREL:%vreg[0-9]+]]<def> = MOVaddr <ga:@internal_var>[TF=1], <ga:@internal_var>[TF=130]
+; CHECK: [[DIFF:%vreg[0-9]+]]<def> = SUBXrr [[INTERNAL_VAR_PCREL]]<kill>, [[GOT_PCREL]]<kill>
+; CHECK: STRWroX [[VAL]]<kill>, [[GOT]], [[DIFF]]<kill>, 0, 0
+  store i32 %val, i32* @internal_var
+
+  ret void
+}

test/CodeGen/AArch64/pagerando-optimizer.ll

+; RUN: llc < %s -march=aarch64 -relocation-model=pip -o - | FileCheck %s
+
+; CHECK-LABEL: .text
+; CHECK-LABEL: wrapper:
+define void @wrapper() pagerando_wrapper { ret void }
+
+; CHECK-LABEL: section .text.bin_1
+; CHECK-LABEL: orig:
+define hidden void @orig() pagerando_binned { ret void }
+
+; CHECK-LABEL: user:
+define void @user() pagerando_binned {
+  call void @wrapper()
+
+; CHECK-NOT: .text.bin_1
+; CHECK: bl orig
+  call void @orig()
+
+  ret void
+}
+
+; CHECK-LABEL: .section .pot
+; CHECK-LABEL: _PAGE_OFFSET_TABLE_:
+; CHECK-NOT: .text.bin_1