[AArch64][GlobalISel] Split G_GLOBAL_VALUE into ADRP + G_ADD_LOW and optimize.

[aemerson]

Summary

The concept of G_GLOBAL_VALUE is nice and simple, but always using it as the representation for global var addressing until selection time creates some problems in optimizing accesses in certain code/relocation models.

The problem comes from trying to optimize adrp -> add -> load/store sequences in the most common "small" code model. These accesses can be optimized into an adrp -> load with the add offset being folded into the load's immediate field. If we try to keep all global var references as a single generic instruction then by the time we get to the complex operand trying to match these, we end up generating an adrp at the point of use. The real issue here is that we don't have any form of CSE during selection, so the code size will bloat from many redundant adrp's.

This patch custom legalizes small code mode non-GOT G_GLOBALs into target ADRP and a new "target specific generic opcode" G_ADD_LOW. We also teach the localizer to localize these instructions via the custom hook that was added recently. Finally, the complex pattern for indexed loads/stores is extended to try to fold these G_ADD_LOW instructions into the load immediate.

On -O0 CTMark, we see a 0.8% geomean code size improvement. We should also see some minor performance improvements too.

[paquette]

LGTM

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

 
 // Unlike X86, we let frame lowering assign offsets to all catch objects.
 bool AArch64TargetLowering::needsFixedCatchObjects() const {
   return false;
 }
 
 bool AArch64TargetLowering::shouldLocalize(
     const MachineInstr &MI, const TargetTransformInfo *TTI) const {
-  if (MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
+  switch (MI.getOpcode()) {
+  case TargetOpcode::G_GLOBAL_VALUE: {
     // On Darwin, TLS global vars get selected into function calls, which
     // we don't want localized, as they can get moved into the middle of a
     // another call sequence.
     const GlobalValue &GV = *MI.getOperand(1).getGlobal();
     if (GV.isThreadLocal() && Subtarget->isTargetMachO())
       return false;
+    break;
+  }
+  // If we legalized G_GLOBAL_VALUE into ADRP + G_ADD_LOW, mark both as being
+  // localizable.
+  case AArch64::ADRP:
+  case AArch64::G_ADD_LOW:
+    return true;
+  default:
+    break;
   }
   return TargetLoweringBase::shouldLocalize(MI, TTI);
 }

llvm/lib/Target/AArch64/AArch64InstrGISel.td

+//=----- AArch64InstrGISel.td - AArch64 GISel target pseudos -*- tablegen -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// AArch64 GlobalISel target pseudo instruction definitions. This is kept
+// separately from the other tablegen files for organizational purposes, but
+// share the same infrastructure.
+//
+//===----------------------------------------------------------------------===//
+
+
+class AArch64GenericInstruction : GenericInstruction {
+  let Namespace = "AArch64";
+}
+
+// A pseudo to represent a relocatable add instruction as part of address
+// computation.
+def G_ADD_LOW : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type1:$src, type2:$imm);
+  let hasSideEffects = 0;
+}

llvm/lib/Target/AArch64/AArch64InstrInfo.td

let AddedComplexity = 10 in {
   def : dot_v16i8<UDOTv16i8, AArch64umull, zext>;
 
   // FIXME: add patterns to generate vector by element dot product.
   // FIXME: add SVE dot-product patterns.
 }
 
 include "AArch64InstrAtomics.td"
 include "AArch64SVEInstrInfo.td"
+
+include "AArch64InstrGISel.td"

llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp

private:
   }
   ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
     return selectAddrModeUnscaled(Root, 8);
   }
   ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
     return selectAddrModeUnscaled(Root, 16);
   }
 
+  /// Helper to try to fold in a GISEL_ADD_LOW into an immediate, to be used
+  /// from complex pattern matchers like selectAddrModeIndexed().
+  ComplexRendererFns tryFoldAddLowIntoImm(MachineInstr &RootDef, unsigned Size,
+                                          MachineRegisterInfo &MRI) const;
+
   ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
                                            unsigned Size) const;
   template <int Width>
   ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
     return selectAddrModeIndexed(Root, Width / 8);
   }
 
   bool isWorthFoldingIntoExtendedReg(MachineInstr &MI,
if (AmtMI->getOpcode() != TargetOpcode::G_CONSTANT) {
       I.getOperand(2).setReg(Trunc.getReg(0));
     }
     return true;
   }
   case TargetOpcode::G_STORE:
     return contractCrossBankCopyIntoStore(I, MRI);
   case TargetOpcode::G_PTR_ADD:
     return convertPtrAddToAdd(I, MRI);
+  case TargetOpcode::G_LOAD: {
+    // For scalar loads of pointers, we try to convert the dest type from p0
+    // to s64 so that our imported patterns can match. Like with the G_PTR_ADD
+    // conversion, this should be ok because all users should have been
+    // selected already, so the type doesn't matter for them.
+    Register DstReg = I.getOperand(0).getReg();
+    const LLT DstTy = MRI.getType(DstReg);
+    if (!DstTy.isPointer())
+      return false;
+    MRI.setType(DstReg, LLT::scalar(64));
+    return true;
+  }
   default:
     return false;
   }
 }
 
 /// This lowering tries to look for G_PTR_ADD instructions and then converts
 /// them to a standard G_ADD with a COPY on the source.
 ///
bool AArch64InstructionSelector::select(MachineInstr &I) {
   if (Subtarget->requiresStrictAlign()) {
     // We don't support this feature yet.
     LLVM_DEBUG(dbgs() << "AArch64 GISel does not support strict-align yet\n");
     return false;
   }
 
   unsigned Opcode = I.getOpcode();
   // G_PHI requires same handling as PHI
-  if (!isPreISelGenericOpcode(Opcode) || Opcode == TargetOpcode::G_PHI) {
+  if (!I.isPreISelOpcode() || Opcode == TargetOpcode::G_PHI) {
     // Certain non-generic instructions also need some special handling.
 
     if (Opcode ==  TargetOpcode::LOAD_STACK_GUARD)
       return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
 
     if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
       const Register DefReg = I.getOperand(0).getReg();
       const LLT DefTy = MRI.getType(DefReg);
bool AArch64InstructionSelector::select(MachineInstr &I) {
   case TargetOpcode::G_BRINDIRECT: {
     I.setDesc(TII.get(AArch64::BR));
     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
   }
 
   case TargetOpcode::G_BRJT:
     return selectBrJT(I, MRI);
 
+  case AArch64::G_ADD_LOW: {
+    I.setDesc(TII.get(AArch64::ADDXri));
+    I.addOperand(MachineOperand::CreateImm(0));
+    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
+  }
+
   case TargetOpcode::G_BSWAP: {
     // Handle vector types for G_BSWAP directly.
     Register DstReg = I.getOperand(0).getReg();
     LLT DstTy = MRI.getType(DstReg);
 
     // We should only get vector types here; everything else is handled by the
     // importer right now.
     if (!DstTy.isVector() || DstTy.getSizeInBits() > 128) {
if (RHSC >= -256 && RHSC < 256) {
     return {{
         [=](MachineInstrBuilder &MIB) { MIB.add(Base); },
         [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
     }};
   }
   return None;
 }
 
+InstructionSelector::ComplexRendererFns
+AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
+                                                 unsigned Size,
+                                                 MachineRegisterInfo &MRI) const {
+  if (RootDef.getOpcode() != AArch64::G_ADD_LOW)
+    return None;
+  MachineInstr &Adrp = *MRI.getVRegDef(RootDef.getOperand(1).getReg());
+  if (Adrp.getOpcode() != AArch64::ADRP)
+    return None;
+
+  // TODO: add heuristics like isWorthFoldingADDlow() from SelectionDAG.
+  // TODO: Need to check GV's offset % size if doing offset folding into globals.
+  assert(Adrp.getOperand(1).getOffset() == 0 && "Unexpected offset in global");
+  auto GV = Adrp.getOperand(1).getGlobal();
+  if (GV->isThreadLocal())
+    return None;
+
+  unsigned Alignment = GV->getAlignment();
+  Type *Ty = GV->getValueType();
+  auto &MF = *RootDef.getParent()->getParent();
+  if (Alignment == 0 && Ty->isSized())
+    Alignment = MF.getDataLayout().getABITypeAlignment(Ty);
+
+  if (Alignment < Size)
+    return None;
+
+  unsigned OpFlags = STI.ClassifyGlobalReference(GV, MF.getTarget());
+  MachineIRBuilder MIRBuilder(RootDef);
+  Register AdrpReg = Adrp.getOperand(0).getReg();
+  return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(AdrpReg); },
+           [=](MachineInstrBuilder &MIB) {
+             MIB.addGlobalAddress(GV, /* Offset */ 0,
+                                  OpFlags | AArch64II::MO_PAGEOFF |
+                                      AArch64II::MO_NC);
+           }}};
+}
+
 /// Select a "register plus scaled unsigned 12-bit immediate" address.  The
 /// "Size" argument is the size in bytes of the memory reference, which
 /// determines the scale.
 InstructionSelector::ComplexRendererFns
 AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
                                                   unsigned Size) const {
-  MachineRegisterInfo &MRI =
-      Root.getParent()->getParent()->getParent()->getRegInfo();
+  MachineFunction &MF = *Root.getParent()->getParent()->getParent();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
 
   if (!Root.isReg())
     return None;
 
   MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
   if (!RootDef)
     return None;
 
   if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
     return {{
         [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
         [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
     }};
   }
 
+  CodeModel::Model CM = MF.getTarget().getCodeModel();
+  // Check if we can fold in the ADD of small code model ADRP + ADD address.
+  if (CM == CodeModel::Small) {
+    auto OpFns = tryFoldAddLowIntoImm(*RootDef, Size, MRI);
+    if (OpFns)
+      return OpFns;
+  }
+
   if (isBaseWithConstantOffset(Root, MRI)) {
     MachineOperand &LHS = RootDef->getOperand(1);
     MachineOperand &RHS = RootDef->getOperand(2);
     MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
     MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
     if (LHSDef && RHSDef) {
       int64_t RHSC = (int64_t)RHSDef->getOperand(1).getCImm()->getZExtValue();
       unsigned Scale = Log2_32(Size);

llvm/lib/Target/AArch64/AArch64LegalizerInfo.h

private:
   bool legalizeVaArg(MachineInstr &MI, MachineRegisterInfo &MRI,
                      MachineIRBuilder &MIRBuilder) const;
   bool legalizeLoadStore(MachineInstr &MI, MachineRegisterInfo &MRI,
                          MachineIRBuilder &MIRBuilder,
                          GISelChangeObserver &Observer) const;
   bool legalizeShlAshrLshr(MachineInstr &MI, MachineRegisterInfo &MRI,
                            MachineIRBuilder &MIRBuilder,
                            GISelChangeObserver &Observer) const;
+
+  bool legalizeSmallCMGlobalValue(MachineInstr &MI, MachineRegisterInfo &MRI,
+                                  MachineIRBuilder &MIRBuilder,
+                                  GISelChangeObserver &Observer) const;
+  const AArch64Subtarget *ST;
 };
 } // End llvm namespace.
 #endif

llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp

 
 #define DEBUG_TYPE "aarch64-legalinfo"
 
 using namespace llvm;
 using namespace LegalizeActions;
 using namespace LegalizeMutations;
 using namespace LegalityPredicates;
 
-AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
+AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
+    : ST(&ST) {
   using namespace TargetOpcode;
   const LLT p0 = LLT::pointer(0, 64);
   const LLT s1 = LLT::scalar(1);
   const LLT s8 = LLT::scalar(8);
   const LLT s16 = LLT::scalar(16);
   const LLT s32 = LLT::scalar(32);
   const LLT s64 = LLT::scalar(64);
   const LLT s128 = LLT::scalar(128);
   const LLT s256 = LLT::scalar(256);
   const LLT s512 = LLT::scalar(512);
   const LLT v16s8 = LLT::vector(16, 8);
   const LLT v8s8 = LLT::vector(8, 8);
   const LLT v4s8 = LLT::vector(4, 8);
   const LLT v8s16 = LLT::vector(8, 16);
   const LLT v4s16 = LLT::vector(4, 16);
   const LLT v2s16 = LLT::vector(2, 16);
   const LLT v2s32 = LLT::vector(2, 32);
   const LLT v4s32 = LLT::vector(4, 32);
   const LLT v2s64 = LLT::vector(2, 64);
   const LLT v2p0 = LLT::vector(2, p0);
 
+  const TargetMachine &TM = ST.getTargetLowering()->getTargetMachine();
+
   // FIXME: support subtargets which have neon/fp-armv8 disabled.
   if (!ST.hasNEON() || !ST.hasFPARMv8()) {
     computeTables();
     return;
   }
 
   getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE})
     .legalFor({p0, s1, s8, s16, s32, s64, v2s32, v4s32, v2s64})
AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
   getActionDefinitionsBuilder(G_SELECT)
       .legalFor({{s32, s1}, {s64, s1}, {p0, s1}})
       .clampScalar(0, s32, s64)
       .widenScalarToNextPow2(0)
       .scalarize(0);
 
   // Pointer-handling
   getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0});
+
+  if (TM.getCodeModel() == CodeModel::Small)
+    getActionDefinitionsBuilder(G_GLOBAL_VALUE).custom();
+  else
     getActionDefinitionsBuilder(G_GLOBAL_VALUE).legalFor({p0});
 
   getActionDefinitionsBuilder(G_PTRTOINT)
       .legalForCartesianProduct({s1, s8, s16, s32, s64}, {p0})
       .maxScalar(0, s64)
       .widenScalarToNextPow2(0, /*Min*/ 8);
 
   getActionDefinitionsBuilder(G_INTTOPTR)
       .unsupportedIf([&](const LegalityQuery &Query) {
case TargetOpcode::G_VAARG:
     return legalizeVaArg(MI, MRI, MIRBuilder);
   case TargetOpcode::G_LOAD:
   case TargetOpcode::G_STORE:
     return legalizeLoadStore(MI, MRI, MIRBuilder, Observer);
   case TargetOpcode::G_SHL:
   case TargetOpcode::G_ASHR:
   case TargetOpcode::G_LSHR:
     return legalizeShlAshrLshr(MI, MRI, MIRBuilder, Observer);
+  case TargetOpcode::G_GLOBAL_VALUE:
+    return legalizeSmallCMGlobalValue(MI, MRI, MIRBuilder, Observer);
   }
 
   llvm_unreachable("expected switch to return");
 }
 
+bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(MachineInstr &MI,
+                                                      MachineRegisterInfo &MRI,
+                                                      MachineIRBuilder &MIRBuilder,
+                                                      GISelChangeObserver &Observer) const {
+  assert(MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
+  // We do this custom legalization to convert G_GLOBAL_VALUE into target ADRP +
+  // G_ADD_LOW instructions.
+  // By splitting this here, we can optimize accesses in the small code model by
+  // folding in the G_ADD_LOW into the load/store offset.
+  auto GV = MI.getOperand(1).getGlobal();
+  if (GV->isThreadLocal())
+    return true; // Don't want to modify TLS vars.
+
+  MIRBuilder.setInstrAndDebugLoc(MI);
+  auto &TM = ST->getTargetLowering()->getTargetMachine();
+  unsigned OpFlags = ST->ClassifyGlobalReference(GV, TM);
+
+  if (OpFlags & AArch64II::MO_GOT)
+    return true;
+
+  Register DstReg = MI.getOperand(0).getReg();
+  auto ADRP = MIRBuilder.buildInstr(AArch64::ADRP, {LLT::pointer(0, 64)}, {})
+                  .addGlobalAddress(GV, 0, OpFlags | AArch64II::MO_PAGE);
+  // Set the regclass on the dest reg too.
+  MRI.setRegClass(ADRP.getReg(0), &AArch64::GPR64RegClass);
+
+  MIRBuilder.buildInstr(AArch64::G_ADD_LOW, {DstReg}, {ADRP})
+      .addGlobalAddress(GV, 0,
+                        OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
+  MI.eraseFromParent();
+  return true;
+}
+
 bool AArch64LegalizerInfo::legalizeIntrinsic(
     MachineInstr &MI, MachineIRBuilder &MIRBuilder,
     GISelChangeObserver &Observer) const {
   switch (MI.getIntrinsicID()) {
   case Intrinsic::memcpy:
   case Intrinsic::memset:
   case Intrinsic::memmove:
     if (createMemLibcall(MIRBuilder, *MIRBuilder.getMRI(), MI) ==

llvm/test/CodeGen/AArch64/GlobalISel/call-translator-variadic-musttail.ll

 ; CHECK-NEXT:    .cfi_offset w21, -40
 ; CHECK-NEXT:    .cfi_offset w22, -48
 ; CHECK-NEXT:    .cfi_offset w23, -56
 ; CHECK-NEXT:    .cfi_offset w24, -64
 ; CHECK-NEXT:    .cfi_offset w25, -72
 ; CHECK-NEXT:    .cfi_offset w26, -80
 ; CHECK-NEXT:    .cfi_offset w27, -88
 ; CHECK-NEXT:    .cfi_offset w28, -96
+; CHECK-NEXT:    mov x27, x8
+; CHECK-NEXT:    adrp x8, _asdf@PAGE
 ; CHECK-NEXT:    mov w19, w0
-; CHECK-NEXT:  Lloh0:
-; CHECK-NEXT:    adrp x0, _asdf@PAGE
-; CHECK-NEXT:  Lloh1:
-; CHECK-NEXT:    add x0, x0, _asdf@PAGEOFF
+; CHECK-NEXT:    add x0, x8, _asdf@PAGEOFF
 ; CHECK-NEXT:    mov x20, x1
 ; CHECK-NEXT:    mov x21, x2
 ; CHECK-NEXT:    mov x22, x3
 ; CHECK-NEXT:    mov x23, x4
 ; CHECK-NEXT:    mov x24, x5
 ; CHECK-NEXT:    mov x25, x6
 ; CHECK-NEXT:    mov x26, x7
 ; CHECK-NEXT:    stp q1, q0, [sp, #96] ; 32-byte Folded Spill
 ; CHECK-NEXT:    stp q3, q2, [sp, #64] ; 32-byte Folded Spill
 ; CHECK-NEXT:    stp q5, q4, [sp, #32] ; 32-byte Folded Spill
 ; CHECK-NEXT:    stp q7, q6, [sp] ; 32-byte Folded Spill
-; CHECK-NEXT:    mov x27, x8
 ; CHECK-NEXT:    bl _puts
 ; CHECK-NEXT:    ldp q1, q0, [sp, #96] ; 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q3, q2, [sp, #64] ; 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q5, q4, [sp, #32] ; 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q7, q6, [sp] ; 32-byte Folded Reload
 ; CHECK-NEXT:    mov w0, w19
 ; CHECK-NEXT:    mov x1, x20
 ; CHECK-NEXT:    mov x2, x21
 ; CHECK-NEXT:    mov x3, x22
 ; CHECK-NEXT:    mov x4, x23
 ; CHECK-NEXT:    mov x5, x24
 ; CHECK-NEXT:    mov x6, x25
 ; CHECK-NEXT:    mov x7, x26
 ; CHECK-NEXT:    mov x8, x27
 ; CHECK-NEXT:    ldp x29, x30, [sp, #208] ; 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x20, x19, [sp, #192] ; 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x22, x21, [sp, #176] ; 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x24, x23, [sp, #160] ; 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x26, x25, [sp, #144] ; 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x28, x27, [sp, #128] ; 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #224 ; =224
 ; CHECK-NEXT:    b _musttail_variadic_callee
-; CHECK-NEXT:    .loh AdrpAdd Lloh0, Lloh1
   call void @puts(i8* getelementptr ([4 x i8], [4 x i8]* @asdf, i32 0, i32 0))
   %r = musttail call i32 (i32, ...) @musttail_variadic_callee(i32 %arg0, ...)
   ret i32 %r
 }
 
 ; Test musttailing with a varargs call in the block. Test that we spill and
 ; reload all arguments in the variadic argument pack.
 declare void @llvm.va_start(i8*) nounwind

 ; CHECK-LABEL: h_thunk:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    ldrb w9, [x0]
 ; CHECK-NEXT:    tbz w9, #0, LBB5_2
 ; CHECK-NEXT:  ; %bb.1: ; %then
 ; CHECK-NEXT:    ldr x9, [x0, #8]
 ; CHECK-NEXT:    br x9
 ; CHECK-NEXT:  LBB5_2: ; %else
-; CHECK-NEXT:  Lloh2:
+; CHECK-NEXT:  Lloh0:
 ; CHECK-NEXT:    adrp x10, _g@GOTPAGE
 ; CHECK-NEXT:    ldr x9, [x0, #16]
-; CHECK-NEXT:  Lloh3:
+; CHECK-NEXT:  Lloh1:
 ; CHECK-NEXT:    ldr x10, [x10, _g@GOTPAGEOFF]
 ; CHECK-NEXT:    mov w11, #42
-; CHECK-NEXT:  Lloh4:
+; CHECK-NEXT:  Lloh2:
 ; CHECK-NEXT:    str w11, [x10]
 ; CHECK-NEXT:    br x9
-; CHECK-NEXT:    .loh AdrpLdrGotStr Lloh2, Lloh3, Lloh4
+; CHECK-NEXT:    .loh AdrpLdrGotStr Lloh0, Lloh1, Lloh2
   %cond_p = getelementptr %struct.Foo, %struct.Foo* %this, i32 0, i32 0
   %cond = load i1, i1* %cond_p
   br i1 %cond, label %then, label %else
 
 then:
   %a_p = getelementptr %struct.Foo, %struct.Foo* %this, i32 0, i32 1
   %a_i8 = load i8*, i8** %a_p
   %a = bitcast i8* %a_i8 to void (%struct.Foo*, ...)*

llvm/test/CodeGen/AArch64/GlobalISel/combine-ext-debugloc.mir

registers:
   - { id: 5, class: _, preferred-register: '' }
   - { id: 6, class: _, preferred-register: '' }
   - { id: 7, class: _, preferred-register: '' }
   - { id: 8, class: _, preferred-register: '' }
   - { id: 9, class: _, preferred-register: '' }
 body:             |
   bb.1.entry:
     %0:_(s8) = G_IMPLICIT_DEF
-    %4:_(p0) = G_GLOBAL_VALUE @.str, debug-location !DILocation(line: 0, scope: !4)
+    %4:_(p0) = COPY $x0
     %10:_(p0) = G_IMPLICIT_DEF debug-location !DILocation(line: 0, scope: !4)
     %1:_(s1) = G_TRUNC %0(s8)
     %2:_(s32) = G_ZEXT %1(s1)
     ADJCALLSTACKDOWN 8, 0, implicit-def $sp, implicit $sp, debug-location !8
     $x0 = COPY %4(p0), debug-location !8
     %5:_(p0) = COPY $sp, debug-location !8
     %6:_(s64) = G_CONSTANT i64 0
     %7:_(p0) = G_PTR_ADD %5, %6(s64), debug-location !8

llvm/test/CodeGen/AArch64/GlobalISel/legalize-blockaddress.mir

 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
 body:             |
   ; CHECK-LABEL: name: test_blockaddress
   ; CHECK: bb.0 (%ir-block.0):
   ; CHECK:   successors: %bb.1(0x80000000)
   ; CHECK:   [[BLOCK_ADDR:%[0-9]+]]:_(p0) = G_BLOCK_ADDR blockaddress(@test_blockaddress, %ir-block.block)
-  ; CHECK:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @addr
-  ; CHECK:   G_STORE [[BLOCK_ADDR]](p0), [[GV]](p0) :: (store 8 into @addr)
+  ; CHECK:   [[ADRP:%[0-9]+]]:gpr64(p0) = ADRP target-flags(aarch64-page) @addr
+  ; CHECK:   [[ADD_LOW:%[0-9]+]]:_(p0) = G_ADD_LOW [[ADRP]](p0), target-flags(aarch64-pageoff, aarch64-nc) @addr
+  ; CHECK:   G_STORE [[BLOCK_ADDR]](p0), [[ADD_LOW]](p0) :: (store 8 into @addr)
   ; CHECK:   G_BRINDIRECT [[BLOCK_ADDR]](p0)
   ; CHECK: bb.1.block (address-taken):
   ; CHECK:   RET_ReallyLR
   bb.1 (%ir-block.0):
     %0:_(p0) = G_BLOCK_ADDR blockaddress(@test_blockaddress, %ir-block.block)
     %1:_(p0) = G_GLOBAL_VALUE @addr
     G_STORE %0(p0), %1(p0) :: (store 8 into @addr)
     G_BRINDIRECT %0(p0)
 
   bb.2.block (address-taken):
     RET_ReallyLR
 
 ...
 

llvm/test/CodeGen/AArch64/GlobalISel/legalize-constant.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -O0 -run-pass=legalizer %s -o - | FileCheck %s
 
 --- |
   target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
   target triple = "aarch64--"
   define void @test_constant() {
   entry:
     ret void
   }
   define void @test_fconstant() {
   entry:
     ret void
   }
-  @var = global i8 0
-  define i8* @test_global() { ret i8* undef }
 ...
 
 ---
 name:            test_constant
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
   - { id: 2, class: _ }
bb.0.entry:
     %0(s32) = G_FCONSTANT float 1.0
     $w0 = COPY %0
     %1(s64) = G_FCONSTANT double 2.0
     $x0 = COPY %1
     %2(s16) = G_FCONSTANT half 0.0
     %3:_(s32) = G_ANYEXT %2
     $w0 = COPY %3
 ...
-
----
-name:            test_global
-registers:
-  - { id: 0, class: _ }
-body: |
-  bb.0:
-
-    ; CHECK-LABEL: name: test_global
-    ; CHECK: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @var
-    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[GV]](p0)
-    ; CHECK: $x0 = COPY [[PTRTOINT]](s64)
-    %0(p0) = G_GLOBAL_VALUE @var
-    %1:_(s64) = G_PTRTOINT %0
-    $x0 = COPY %1
-...

llvm/test/CodeGen/AArch64/GlobalISel/legalize-global.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -run-pass=legalizer --relocation-model=pic %s -o - | FileCheck %s --check-prefix=PIC
+# RUN: llc -O0 -run-pass=legalizer --code-model=large %s -o - | FileCheck %s --check-prefix=CMLARGE
+
+--- |
+  target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64--"
+  @var = external global i8
+  define i8* @test_global() { ret i8* undef }
+...
+---
+name:            test_global
+registers:
+  - { id: 0, class: _ }
+body: |
+  bb.0:
+
+    ; We don't want to lower to G_ADD_LOW when we need a GOT access, or when the code
+    ; model isn't 'Small'.
+
+    ; CHECK-LABEL: name: test_global
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64(p0) = ADRP target-flags(aarch64-page) @var
+    ; CHECK: [[ADD_LOW:%[0-9]+]]:_(p0) = G_ADD_LOW [[ADRP]](p0), target-flags(aarch64-pageoff, aarch64-nc) @var
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[ADD_LOW]](p0)
+    ; CHECK: $x0 = COPY [[PTRTOINT]](s64)
+    ; PIC-LABEL: name: test_global
+    ; PIC: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @var
+    ; PIC: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[GV]](p0)
+    ; PIC: $x0 = COPY [[PTRTOINT]](s64)
+    ; CMLARGE-LABEL: name: test_global
+    ; CMLARGE: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @var
+    ; CMLARGE: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[GV]](p0)
+    ; CMLARGE: $x0 = COPY [[PTRTOINT]](s64)
+    %0(p0) = G_GLOBAL_VALUE @var
+    %1:_(s64) = G_PTRTOINT %0
+    $x0 = COPY %1
+...

llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir

 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 #
 # DEBUG-NEXT: G_FRAME_INDEX (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 #
 # DEBUG-NEXT: G_GLOBAL_VALUE (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_EXTRACT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_UNMERGE_VALUES (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected

llvm/test/CodeGen/AArch64/GlobalISel/localizer.mir

if.then:
     store i32 3, i32* @var1, align 4
     store i32 2, i32* @var3, align 4
     store i32 3, i32* @var1, align 4
     br label %if.end
 
   if.end:
     ret i32 0
   }
+  define i32 @adrp_add() {
+  entry:
+    %0 = load i32, i32* @var1, align 4
+    %cmp = icmp eq i32 %0, 1
+    br i1 %cmp, label %if.then, label %if.end
+
+  if.then:
+    store i32 2, i32* @var2, align 4
+    store i32 3, i32* @var1, align 4
+    store i32 2, i32* @var3, align 4
+    store i32 3, i32* @var1, align 4
+    br label %if.end
+
+  if.end:
+    ret i32 0
+  }
 
   define void @test_inttoptr() { ret void }
   define void @many_local_use_intra_block() { ret void }
 
 ...
 
 ---
 name:            local_use
bb.2.if.then:
     G_STORE %6(s32), %1(p0) :: (store 4 into @var1)
 
   bb.3.if.end:
     $w0 = COPY %8(s32)
     RET_ReallyLR implicit $w0
 
 ...
 ---
+name:            adrp_add
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: adrp_add
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[ADRP:%[0-9]+]]:gpr64(p0) = ADRP target-flags(aarch64-page) @var1
+  ; CHECK:   %addlow1:gpr(p0) = G_ADD_LOW [[ADRP]](p0), target-flags(aarch64-pageoff, aarch64-nc) @var1
+  ; CHECK:   [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 2
+  ; CHECK:   [[ADRP1:%[0-9]+]]:gpr64(p0) = ADRP target-flags(aarch64-page) @var2
+  ; CHECK:   %addlow2:gpr(p0) = G_ADD_LOW [[ADRP1]](p0), target-flags(aarch64-pageoff, aarch64-nc) @var2
+  ; CHECK:   [[C1:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 3
+  ; CHECK:   [[ADRP2:%[0-9]+]]:gpr64(p0) = ADRP target-flags(aarch64-page) @var3
+  ; CHECK:   %addlow3:gpr(p0) = G_ADD_LOW [[ADRP2]](p0), target-flags(aarch64-pageoff, aarch64-nc) @var3
+  ; CHECK:   [[C2:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[LOAD:%[0-9]+]]:gpr(s32) = G_LOAD [[ADRP]](p0) :: (load 4 from @var1)
+  ; CHECK:   [[C3:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 1
+  ; CHECK:   [[ICMP:%[0-9]+]]:gpr(s32) = G_ICMP intpred(eq), [[LOAD]](s32), [[C3]]
+  ; CHECK:   [[TRUNC:%[0-9]+]]:gpr(s1) = G_TRUNC [[ICMP]](s32)
+  ; CHECK:   G_BRCOND [[TRUNC]](s1), %bb.1
+  ; CHECK:   G_BR %bb.2
+  ; CHECK: bb.1.if.then:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   [[ADRP3:%[0-9]+]]:gpr64(p0) = ADRP target-flags(aarch64-page) @var2
+  ; CHECK:   [[ADD_LOW:%[0-9]+]]:gpr(p0) = G_ADD_LOW [[ADRP3]](p0), target-flags(aarch64-pageoff, aarch64-nc) @var2
+  ; CHECK:   [[C4:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 2
+  ; CHECK:   G_STORE [[C4]](s32), [[ADD_LOW]](p0) :: (store 4 into @var2)
+  ; CHECK:   [[C5:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 3
+  ; CHECK:   [[ADRP4:%[0-9]+]]:gpr64(p0) = ADRP target-flags(aarch64-page) @var1
+  ; CHECK:   [[ADD_LOW1:%[0-9]+]]:gpr(p0) = G_ADD_LOW [[ADRP4]](p0), target-flags(aarch64-pageoff, aarch64-nc) @var1
+  ; CHECK:   G_STORE [[C5]](s32), [[ADD_LOW1]](p0) :: (store 4 into @var1)
+  ; CHECK:   [[ADRP5:%[0-9]+]]:gpr64(p0) = ADRP target-flags(aarch64-page) @var3
+  ; CHECK:   [[ADD_LOW2:%[0-9]+]]:gpr(p0) = G_ADD_LOW [[ADRP5]](p0), target-flags(aarch64-pageoff, aarch64-nc) @var3
+  ; CHECK:   G_STORE [[C4]](s32), [[ADD_LOW2]](p0) :: (store 4 into @var3)
+  ; CHECK:   G_STORE [[C5]](s32), [[ADD_LOW1]](p0) :: (store 4 into @var1)
+  ; CHECK: bb.2.if.end:
+  ; CHECK:   [[C6:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 0
+  ; CHECK:   $w0 = COPY [[C6]](s32)
+  ; CHECK:   RET_ReallyLR implicit $w0
+
+  ; Some of these instructions are dead.
+  bb.1.entry:
+    %1:gpr64(p0) = ADRP target-flags(aarch64-page) @var1
+    %addlow1:gpr(p0) = G_ADD_LOW %1(p0), target-flags(aarch64-pageoff, aarch64-nc) @var1
+    %2:gpr(s32) = G_CONSTANT i32 1
+    %4:gpr(s32) = G_CONSTANT i32 2
+    %5:gpr64(p0) = ADRP target-flags(aarch64-page) @var2
+    %addlow2:gpr(p0) = G_ADD_LOW %5(p0), target-flags(aarch64-pageoff, aarch64-nc) @var2
+    %6:gpr(s32) = G_CONSTANT i32 3
+    %7:gpr64(p0) = ADRP target-flags(aarch64-page) @var3
+    %addlow3:gpr(p0) = G_ADD_LOW %7(p0), target-flags(aarch64-pageoff, aarch64-nc) @var3
+    %8:gpr(s32) = G_CONSTANT i32 0
+    %0:gpr(s32) = G_LOAD %1(p0) :: (load 4 from @var1)
+    %9:gpr(s32) = G_ICMP intpred(eq), %0(s32), %2
+    %3:gpr(s1) = G_TRUNC %9(s32)
+    G_BRCOND %3(s1), %bb.2
+    G_BR %bb.3
+
+  bb.2.if.then:
+    G_STORE %4(s32), %addlow2(p0) :: (store 4 into @var2)
+    G_STORE %6(s32), %addlow1(p0) :: (store 4 into @var1)
+    G_STORE %4(s32), %addlow3(p0) :: (store 4 into @var3)
+    G_STORE %6(s32), %addlow1(p0) :: (store 4 into @var1)
+
+  bb.3.if.end:
+    $w0 = COPY %8(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
 name:            test_inttoptr
 alignment:       4
 legalized:       true
 regBankSelected: true
 tracksRegLiveness: true
 body:             |
   ; CHECK-LABEL: name: test_inttoptr
   ; CHECK: bb.0:

llvm/test/CodeGen/AArch64/arm64-custom-call-saved-reg.ll

 ; RUN: -mattr=+call-saved-x11 \
 ; RUN: -mattr=+call-saved-x12 \
 ; RUN: -mattr=+call-saved-x13 \
 ; RUN: -mattr=+call-saved-x14 \
 ; RUN: -mattr=+call-saved-x15 \
 ; RUN: -mattr=+call-saved-x18 \
 ; RUN: -global-isel \
 ; RUN: -o - %s | FileCheck %s \
-; RUN: --check-prefix=CHECK-SAVED-ALL
+; RUN: --check-prefix=CHECK-SAVED-ALL-GISEL
 
 ; Used to exhaust the supply of GPRs.
 @var = global [30 x i64] zeroinitializer
 
 ; Check that callee preserves additional CSRs.
 define void @callee() {
 ; CHECK-LABEL: callee
 

 ; CHECK-SAVED-ALL-DAG: ldr x10
 ; CHECK-SAVED-ALL-DAG: ldr x11
 ; CHECK-SAVED-ALL-DAG: ldr x12
 ; CHECK-SAVED-ALL-DAG: ldr x13
 ; CHECK-SAVED-ALL-DAG: ldr x14
 ; CHECK-SAVED-ALL-DAG: ldr x15
 ; CHECK-SAVED-ALL-DAG: ldr x18
 
+; CHECK-SAVED-ALL-GISEL: adrp x16, var
+; CHECK-SAVED-ALL-GISEL-DAG: ldr x8
+; CHECK-SAVED-ALL-GISEL-DAG: ldr x9
+; CHECK-SAVED-ALL-GISEL-DAG: ldr x10
+; CHECK-SAVED-ALL-GISEL-DAG: ldr x11
+; CHECK-SAVED-ALL-GISEL-DAG: ldr x12
+; CHECK-SAVED-ALL-GISEL-DAG: ldr x13
+; CHECK-SAVED-ALL-GISEL-DAG: ldr x14
+; CHECK-SAVED-ALL-GISEL-DAG: ldr x15
+; CHECK-SAVED-ALL-GISEL-DAG: ldr x18
+
   call void @callee()
 ; CHECK: bl callee
 
   store volatile [30 x i64] %val, [30 x i64]* @var
 ; CHECK-SAVED-ALL-DAG: str x9
 ; CHECK-SAVED-ALL-DAG: str x10
 ; CHECK-SAVED-ALL-DAG: str x11
 ; CHECK-SAVED-ALL-DAG: str x12
 ; CHECK-SAVED-ALL-DAG: str x13
 ; CHECK-SAVED-ALL-DAG: str x14
 ; CHECK-SAVED-ALL-DAG: str x15
 ; CHECK-SAVED-ALL-DAG: str x18
 
   ret void
 }

llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll

 ; FALLBACK-NOT: remark:{{.*}}test_load_i8
 define void @test_load_i8(i8* %addr) {
 ; CHECK-LABEL: test_load_i8:
 ; CHECK: ldxrb w[[LOADVAL:[0-9]+]], [x0]
 ; CHECK-NOT: uxtb
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
-; FIXME: GlobalISel doesn't fold ands/adds into load/store addressing modes
-; right now/ So, we won't get the :lo12:var.
 ; GISEL-LABEL: test_load_i8:
 ; GISEL: ldxrb w[[LOADVAL:[0-9]+]], [x0]
 ; GISEL-NOT: uxtb
-; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
   %val = call i64 @llvm.aarch64.ldxr.p0i8(i8* %addr)
   %shortval = trunc i64 %val to i8
   %extval = zext i8 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_load_i16
 define void @test_load_i16(i16* %addr) {
 ; CHECK-LABEL: test_load_i16:
 ; CHECK: ldxrh w[[LOADVAL:[0-9]+]], [x0]
 ; CHECK-NOT: uxth
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
 ; GISEL-LABEL: test_load_i16:
 ; GISEL: ldxrh w[[LOADVAL:[0-9]+]], [x0]
 ; GISEL-NOT: uxtb
-; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
   %val = call i64 @llvm.aarch64.ldxr.p0i16(i16* %addr)
   %shortval = trunc i64 %val to i16
   %extval = zext i16 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_load_i32
 define void @test_load_i32(i32* %addr) {
 ; CHECK-LABEL: test_load_i32:
 ; CHECK: ldxr w[[LOADVAL:[0-9]+]], [x0]
 ; CHECK-NOT: uxtw
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
 ; GISEL-LABEL: test_load_i32:
 ; GISEL: ldxr w[[LOADVAL:[0-9]+]], [x0]
 ; GISEL-NOT: uxtb
-; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
   %val = call i64 @llvm.aarch64.ldxr.p0i32(i32* %addr)
   %shortval = trunc i64 %val to i32
   %extval = zext i32 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_load_i64
 define void @test_load_i64(i64* %addr) {
 ; CHECK-LABEL: test_load_i64:
 ; CHECK: ldxr x[[LOADVAL:[0-9]+]], [x0]
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
 ; GISEL-LABEL: test_load_i64:
 ; GISEL: ldxr x[[LOADVAL:[0-9]+]], [x0]
 ; GISEL-NOT: uxtb
-; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
   %val = call i64 @llvm.aarch64.ldxr.p0i64(i64* %addr)
   store i64 %val, i64* @var, align 8
   ret void
 }
 
 
 declare i64 @llvm.aarch64.ldxr.p0i8(i8*) nounwind
 declare i64 @llvm.aarch64.ldxr.p0i16(i16*) nounwind

 ; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i8
 define void @test_load_acquire_i8(i8* %addr) {
 ; CHECK-LABEL: test_load_acquire_i8:
 ; CHECK: ldaxrb w[[LOADVAL:[0-9]+]], [x0]
 ; CHECK-NOT: uxtb
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
-; FIXME: GlobalISel doesn't fold ands/adds into load/store addressing modes
-; right now/ So, we won't get the :lo12:var.
 ; GISEL-LABEL: test_load_acquire_i8:
 ; GISEL: ldaxrb w[[LOADVAL:[0-9]+]], [x0]
-; GISEL-DAG: str x[[LOADVAL]], [{{x[0-9]+}}]
+; GISEL-DAG: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
   %val = call i64 @llvm.aarch64.ldaxr.p0i8(i8* %addr)
   %shortval = trunc i64 %val to i8
   %extval = zext i8 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i16
 define void @test_load_acquire_i16(i16* %addr) {
 ; CHECK-LABEL: test_load_acquire_i16:
 ; CHECK: ldaxrh w[[LOADVAL:[0-9]+]], [x0]
 ; CHECK-NOT: uxth
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
 ; GISEL-LABEL: test_load_acquire_i16:
 ; GISEL: ldaxrh w[[LOADVAL:[0-9]+]], [x0]
-; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
   %val = call i64 @llvm.aarch64.ldaxr.p0i16(i16* %addr)
   %shortval = trunc i64 %val to i16
   %extval = zext i16 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i32
 define void @test_load_acquire_i32(i32* %addr) {
 ; CHECK-LABEL: test_load_acquire_i32:
 ; CHECK: ldaxr w[[LOADVAL:[0-9]+]], [x0]
 ; CHECK-NOT: uxtw
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
 ; GISEL-LABEL: test_load_acquire_i32:
 ; GISEL: ldaxr w[[LOADVAL:[0-9]+]], [x0]
-; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
   %val = call i64 @llvm.aarch64.ldaxr.p0i32(i32* %addr)
   %shortval = trunc i64 %val to i32
   %extval = zext i32 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i64
 define void @test_load_acquire_i64(i64* %addr) {
 ; CHECK-LABEL: test_load_acquire_i64:
 ; CHECK: ldaxr x[[LOADVAL:[0-9]+]], [x0]
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
 ; GISEL-LABEL: test_load_acquire_i64:
 ; GISEL: ldaxr x[[LOADVAL:[0-9]+]], [x0]
-; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
   %val = call i64 @llvm.aarch64.ldaxr.p0i64(i64* %addr)
   store i64 %val, i64* @var, align 8
   ret void
 }
 
 
 declare i64 @llvm.aarch64.ldaxr.p0i8(i8*) nounwind
 declare i64 @llvm.aarch64.ldaxr.p0i16(i16*) nounwind

llvm/test/CodeGen/AArch64/dllimport.ll

define i32 @get_ext() {
   ret i32 %1
 }
 
 ; CHECK-LABEL: get_ext
 ; CHECK: adrp x8, ext
 ; DAG-ISEL: ldr w0, [x8, ext]
 ; FAST-ISEL: add x8, x8, ext
 ; FAST-ISEL: ldr w0, [x8]
-; GLOBAL-ISEL-FALLBACK: add x8, x8, ext
-; GLOBAL-ISEL-FALLBACK: ldr w0, [x8]
+; GLOBAL-ISEL-FALLBACK: ldr w0, [x8, ext]
 ; CHECK: ret
 
 define i32* @get_var_pointer() {
   ret i32* @var
 }
 
 ; CHECK-LABEL: get_var_pointer
 ; CHECK: adrp [[REG1:x[0-9]+]], __imp_var