[AArch64][GlobalISel] Import XRO load/store patterns instead of custom selection

[paquette]

Summary

Instead of using custom C++ in earlySelect for loads and stores, just import the patterns.

Remove earlySelectLoad, since we can just import the work it's doing.

Some minor changes to how ComplexRendererFns are returned for the XRO addressing modes. If you add immediates in two steps, sometimes they are not imported properly and you only end up with one immediate. I'm not sure if this is intentional.

• Update load-addressing-modes.mir to include the instructions we can now import.

• Add a similar test, store-addressing-modes.mir to show which store opcodes we currently import, and show that we can pull in shifts etc.

• Update arm64-fastisel-gep-promote-before-add.ll to use FastISel instead of GISel. This test failed with GISel because GISel folds the gep into the load. The test checks that FastISel doesn't fold non-pointer-width adds into loads. GISel on the other hand, produces a G_CONSTANT of -128 for the add, and then a G_GEP, which must be pointer-width.

Note that we don't get STRBRoX right now. It seems like the importer can't handle FPR8Op:{ *:[Untyped] }:$Rt source operands. So, those are not currently supported.

[aemerson]

Nice, LGTM.

llvm/trunk/lib/Target/AArch64/AArch64InstrFormats.td

 //---
 
 def ro_Xindexed8 : ComplexPattern<i64, 4, "SelectAddrModeXRO<8>", []>;
 def ro_Xindexed16 : ComplexPattern<i64, 4, "SelectAddrModeXRO<16>", []>;
 def ro_Xindexed32 : ComplexPattern<i64, 4, "SelectAddrModeXRO<32>", []>;
 def ro_Xindexed64 : ComplexPattern<i64, 4, "SelectAddrModeXRO<64>", []>;
 def ro_Xindexed128 : ComplexPattern<i64, 4, "SelectAddrModeXRO<128>", []>;
 
+def gi_ro_Xindexed8 :
+    GIComplexOperandMatcher<s64, "selectAddrModeXRO<8>">,
+    GIComplexPatternEquiv<ro_Xindexed8>;
+def gi_ro_Xindexed16 :
+    GIComplexOperandMatcher<s64, "selectAddrModeXRO<16>">,
+    GIComplexPatternEquiv<ro_Xindexed16>;
+def gi_ro_Xindexed32 :
+    GIComplexOperandMatcher<s64, "selectAddrModeXRO<32>">,
+    GIComplexPatternEquiv<ro_Xindexed32>;
+def gi_ro_Xindexed64 :
+    GIComplexOperandMatcher<s64, "selectAddrModeXRO<64>">,
+    GIComplexPatternEquiv<ro_Xindexed64>;
+def gi_ro_Xindexed128 :
+    GIComplexOperandMatcher<s64, "selectAddrModeXRO<128>">,
+    GIComplexPatternEquiv<ro_Xindexed128>;
+
 def ro_Windexed8 : ComplexPattern<i64, 4, "SelectAddrModeWRO<8>", []>;
 def ro_Windexed16 : ComplexPattern<i64, 4, "SelectAddrModeWRO<16>", []>;
 def ro_Windexed32 : ComplexPattern<i64, 4, "SelectAddrModeWRO<32>", []>;
 def ro_Windexed64 : ComplexPattern<i64, 4, "SelectAddrModeWRO<64>", []>;
 def ro_Windexed128 : ComplexPattern<i64, 4, "SelectAddrModeWRO<128>", []>;
 
 class MemExtendOperand<string Reg, int Width> : AsmOperandClass {
   let Name = "Mem" # Reg # "Extend" # Width;

llvm/trunk/lib/Target/AArch64/AArch64InstructionSelector.cpp

private:
   // A lowering phase that runs before any selection attempts.
 
   void preISelLower(MachineInstr &I) const;
 
   // An early selection function that runs before the selectImpl() call.
   bool earlySelect(MachineInstr &I) const;
 
   bool earlySelectSHL(MachineInstr &I, MachineRegisterInfo &MRI) const;
-  bool earlySelectLoad(MachineInstr &I, MachineRegisterInfo &MRI) const;
 
   /// Eliminate same-sized cross-bank copies into stores before selectImpl().
   void contractCrossBankCopyIntoStore(MachineInstr &I,
                                       MachineRegisterInfo &MRI) const;
 
   bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
                           MachineRegisterInfo &MRI) const;
   bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
private:
   bool isWorthFoldingIntoExtendedReg(MachineInstr &MI,
                                      const MachineRegisterInfo &MRI) const;
   ComplexRendererFns
   selectAddrModeShiftedExtendXReg(MachineOperand &Root,
                                   unsigned SizeInBytes) const;
   ComplexRendererFns selectAddrModeRegisterOffset(MachineOperand &Root) const;
   ComplexRendererFns selectAddrModeXRO(MachineOperand &Root,
                                        unsigned SizeInBytes) const;
+  template <int Width>
+  ComplexRendererFns selectAddrModeXRO(MachineOperand &Root) const {
+    return selectAddrModeXRO(Root, Width / 8);
+  }
 
   ComplexRendererFns selectShiftedRegister(MachineOperand &Root) const;
 
   ComplexRendererFns selectArithShiftedRegister(MachineOperand &Root) const {
     return selectShiftedRegister(Root);
   }
 
   ComplexRendererFns selectLogicalShiftedRegister(MachineOperand &Root) const {
void AArch64InstructionSelector::contractCrossBankCopyIntoStore(
   if (RBI.getRegBank(StoreSrcReg, MRI, TRI) ==
       RBI.getRegBank(DefDstReg, MRI, TRI))
     return;
 
   // We have a cross-bank copy, which is entering a store. Let's fold it.
   I.getOperand(0).setReg(DefDstReg);
 }
 
-bool AArch64InstructionSelector::earlySelectLoad(
-    MachineInstr &I, MachineRegisterInfo &MRI) const {
-  // Try to fold in shifts, etc into the addressing mode of a load.
-  assert(I.getOpcode() == TargetOpcode::G_LOAD && "unexpected op");
-
-  // Don't handle atomic loads/stores yet.
-  auto &MemOp = **I.memoperands_begin();
-  if (MemOp.isAtomic()) {
-    LLVM_DEBUG(dbgs() << "Atomic load/store not supported yet\n");
-    return false;
-  }
-
-  unsigned MemBytes = MemOp.getSize();
-
-  // Only support 64-bit loads for now.
-  if (MemBytes != 8)
-    return false;
-
-  Register DstReg = I.getOperand(0).getReg();
-  const LLT DstTy = MRI.getType(DstReg);
-  // Don't handle vectors.
-  if (DstTy.isVector())
-    return false;
-
-  unsigned DstSize = DstTy.getSizeInBits();
-  // TODO: 32-bit destinations.
-  if (DstSize != 64)
-    return false;
-
-  // Check if we can do any folding from GEPs/shifts etc. into the load.
-  auto ImmFn = selectAddrModeXRO(I.getOperand(1), MemBytes);
-  if (!ImmFn)
-    return false;
-
-  // We can fold something. Emit the load here.
-  MachineIRBuilder MIB(I);
-
-  // Choose the instruction based off the size of the element being loaded, and
-  // whether or not we're loading into a FPR.
-  const RegisterBank &RB = *RBI.getRegBank(DstReg, MRI, TRI);
-  unsigned Opc =
-      RB.getID() == AArch64::GPRRegBankID ? AArch64::LDRXroX : AArch64::LDRDroX;
-  // Construct the load.
-  auto LoadMI = MIB.buildInstr(Opc, {DstReg}, {});
-  for (auto &RenderFn : *ImmFn)
-    RenderFn(LoadMI);
-  LoadMI.addMemOperand(*I.memoperands_begin());
-  I.eraseFromParent();
-  return constrainSelectedInstRegOperands(*LoadMI, TII, TRI, RBI);
-}
-
 bool AArch64InstructionSelector::earlySelect(MachineInstr &I) const {
   assert(I.getParent() && "Instruction should be in a basic block!");
   assert(I.getParent()->getParent() && "Instruction should be in a function!");
 
   MachineBasicBlock &MBB = *I.getParent();
   MachineFunction &MF = *MBB.getParent();
   MachineRegisterInfo &MRI = MF.getRegInfo();
 
   switch (I.getOpcode()) {
   case TargetOpcode::G_SHL:
     return earlySelectSHL(I, MRI);
-  case TargetOpcode::G_LOAD:
-    return earlySelectLoad(I, MRI);
   case TargetOpcode::G_CONSTANT: {
     bool IsZero = false;
     if (I.getOperand(1).isCImm())
       IsZero = I.getOperand(1).getCImm()->getZExtValue() == 0;
     else if (I.getOperand(1).isImm())
       IsZero = I.getOperand(1).getImm() == 0;
 
     if (!IsZero)
AArch64InstructionSelector::selectAddrModeShiftedExtendXReg(
 
   // We are only allowed to shift by LegalShiftVal. This shift value is built
   // into the instruction, so we can't just use whatever we want.
   if (ImmVal != LegalShiftVal)
     return None;
 
   // We can use the LHS of the GEP as the base, and the LHS of the shift as an
   // offset. Signify that we are shifting by setting the shift flag to 1.
-  return {{
-      [=](MachineInstrBuilder &MIB) { MIB.add(Gep->getOperand(1)); },
+  return {{[=](MachineInstrBuilder &MIB) {
+             MIB.addUse(Gep->getOperand(1).getReg());
+           },
            [=](MachineInstrBuilder &MIB) { MIB.addUse(OffsetReg); },
-      [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
-      [=](MachineInstrBuilder &MIB) { MIB.addImm(1); },
-  }};
+           [=](MachineInstrBuilder &MIB) {
+             // Need to add both immediates here to make sure that they are both
+             // added to the instruction.
+             MIB.addImm(0);
+             MIB.addImm(1);
+           }}};
 }
 
 /// This is used for computing addresses like this:
 ///
 /// ldr x1, [x2, x3]
 ///
 /// Where x2 is the base register, and x3 is an offset register.
 ///
AArch64InstructionSelector::selectAddrModeRegisterOffset(
 
   // If this is used more than once, let's not bother folding.
   // TODO: Check if they are memory ops. If they are, then we can still fold
   // without having to recompute anything.
   if (!MRI.hasOneUse(Gep->getOperand(0).getReg()))
     return None;
 
   // Base is the GEP's LHS, offset is its RHS.
-  return {{
-      [=](MachineInstrBuilder &MIB) { MIB.add(Gep->getOperand(1)); },
-      [=](MachineInstrBuilder &MIB) { MIB.add(Gep->getOperand(2)); },
-      [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
-      [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
-  }};
+  return {{[=](MachineInstrBuilder &MIB) {
+             MIB.addUse(Gep->getOperand(1).getReg());
+           },
+           [=](MachineInstrBuilder &MIB) {
+             MIB.addUse(Gep->getOperand(2).getReg());
+           },
+           [=](MachineInstrBuilder &MIB) {
+             // Need to add both immediates here to make sure that they are both
+             // added to the instruction.
+             MIB.addImm(0);
+             MIB.addImm(0);
+           }}};
 }
 
 /// This is intended to be equivalent to selectAddrModeXRO in
 /// AArch64ISelDAGtoDAG. It's used for selecting X register offset loads.
 InstructionSelector::ComplexRendererFns
 AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root,
                                               unsigned SizeInBytes) const {
   MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();

llvm/trunk/test/CodeGen/AArch64/GlobalISel/load-addressing-modes.mir

--- |
   define void @ldrdrox_mul_lhs(i64* %addr) { ret void }
   define void @mul_not_pow_2(i64* %addr) { ret void }
   define void @mul_wrong_pow_2(i64* %addr) { ret void }
   define void @more_than_one_use_shl_1(i64* %addr) { ret void }
   define void @more_than_one_use_shl_2(i64* %addr) { ret void }
   define void @more_than_one_use_shl_lsl_fast(i64* %addr) #1 { ret void }
   define void @more_than_one_use_shl_lsl_slow(i64* %addr) { ret void }
   define void @more_than_one_use_shl_minsize(i64* %addr) #0 { ret void }
+  define void @ldrwrox(i64* %addr) { ret void }
+  define void @ldrsrox(i64* %addr) { ret void }
+  define void @ldrhrox(i64* %addr) { ret void }
+  define void @ldbbrox(i64* %addr) { ret void }
+  define void @ldrqrox(i64* %addr) { ret void }
   attributes #0 = { optsize minsize }
   attributes #1 = { "target-features"="+lsl-fast" }
 ...
 
 ---
 name:            ldrxrox_breg_oreg
 alignment:       2
 legalized:       true
bb.0:
     ; CHECK: RET_ReallyLR implicit $d0
     %0:gpr(p0) = COPY $d0
     %1:gpr(s64) = COPY $x1
     %2:gpr(p0) = G_GEP %0, %1
     %4:fpr(s64) = G_LOAD %2(p0) :: (load 8 from %ir.addr)
     $d0 = COPY %4(s64)
     RET_ReallyLR implicit $d0
 ...
-
 ---
 name:            more_than_one_use
 alignment:       2
 legalized:       true
 regBankSelected: true
 tracksRegLiveness: true
 machineFunctionInfo: {}
 body:             |
bb.0:
     %4:gpr(p0) = G_GEP %3, %2
     %5:gpr(s64) = G_LOAD %4(p0) :: (load 8 from %ir.addr)
     %6:gpr(s64) = G_ADD %2, %1
     %7:gpr(s64) = G_ADD %5, %6
     %8:gpr(s64) = G_PTRTOINT %4
     %9:gpr(s64) = G_ADD %8, %7
     $x2 = COPY %9(s64)
     RET_ReallyLR implicit $x2
+...
+---
+name:            ldrwrox
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: ldrwrox
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[LDRWroX:%[0-9]+]]:gpr32 = LDRWroX [[COPY]], [[COPY1]], 0, 0 :: (load 4 from %ir.addr)
+    ; CHECK: $w2 = COPY [[LDRWroX]]
+    ; CHECK: RET_ReallyLR implicit $w2
+    %0:gpr(p0) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(p0) = G_GEP %0, %1
+    %4:gpr(s32) = G_LOAD %2(p0) :: (load 4 from %ir.addr)
+    $w2 = COPY %4(s32)
+    RET_ReallyLR implicit $w2
+...
+---
+name:            ldrsrox
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $d0, $x1
+    ; CHECK-LABEL: name: ldrsrox
+    ; CHECK: liveins: $d0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $d0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[LDRSroX:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], [[COPY1]], 0, 0 :: (load 4 from %ir.addr)
+    ; CHECK: $s2 = COPY [[LDRSroX]]
+    ; CHECK: RET_ReallyLR implicit $h2
+    %0:gpr(p0) = COPY $d0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(p0) = G_GEP %0, %1
+    %4:fpr(s32) = G_LOAD %2(p0) :: (load 4 from %ir.addr)
+    $s2 = COPY %4(s32)
+    RET_ReallyLR implicit $h2
+...
+---
+name:            ldrhrox
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: ldrhrox
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[LDRHroX:%[0-9]+]]:fpr16 = LDRHroX [[COPY]], [[COPY1]], 0, 0 :: (load 2 from %ir.addr)
+    ; CHECK: $h2 = COPY [[LDRHroX]]
+    ; CHECK: RET_ReallyLR implicit $h2
+    %0:gpr(p0) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(p0) = G_GEP %0, %1
+    %4:fpr(s16) = G_LOAD %2(p0) :: (load 2 from %ir.addr)
+    $h2 = COPY %4(s16)
+    RET_ReallyLR implicit $h2
+...
+---
+name:            ldbbrox
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: ldbbrox
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[LDRBBroX:%[0-9]+]]:gpr32 = LDRBBroX [[COPY]], [[COPY1]], 0, 0 :: (load 1 from %ir.addr)
+    ; CHECK: $w2 = COPY [[LDRBBroX]]
+    ; CHECK: RET_ReallyLR implicit $w2
+    %0:gpr(p0) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(p0) = G_GEP %0, %1
+    %4:gpr(s32) = G_LOAD %2(p0) :: (load 1 from %ir.addr)
+    $w2 = COPY %4(s32)
+    RET_ReallyLR implicit $w2
+...
+---
+name:            ldrqrox
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $d0, $x1
+    ; CHECK-LABEL: name: ldrqrox
+    ; CHECK: liveins: $d0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $d0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[LDRQroX:%[0-9]+]]:fpr128 = LDRQroX [[COPY]], [[COPY1]], 0, 0 :: (load 16 from %ir.addr)
+    ; CHECK: $q0 = COPY [[LDRQroX]]
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:gpr(p0) = COPY $d0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(p0) = G_GEP %0, %1
+    %4:fpr(<2 x s64>) = G_LOAD %2(p0) :: (load 16 from %ir.addr)
+    $q0 = COPY %4(<2 x s64>)
+    RET_ReallyLR implicit $q0

llvm/trunk/test/CodeGen/AArch64/GlobalISel/store-addressing-modes.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64-unknown-unknown -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+  define void @strxrox(i64* %addr) { ret void }
+  define void @strdrox(i64* %addr) { ret void }
+  define void @strwrox(i64* %addr) { ret void }
+  define void @strsrox(i64* %addr) { ret void }
+  define void @strhrox(i64* %addr) { ret void }
+  define void @strqrox(i64* %addr) { ret void }
+  define void @shl(i64* %addr) { ret void }
+...
+
+---
+name:            strxrox
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0, $x1, $x2
+    ; CHECK-LABEL: name: strxrox
+    ; CHECK: liveins: $x0, $x1, $x2
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[COPY2:%[0-9]+]]:gpr64 = COPY $x2
+    ; CHECK: STRXroX [[COPY2]], [[COPY]], [[COPY1]], 0, 0 :: (store 8 into %ir.addr)
+    %0:gpr(p0) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %ptr:gpr(p0) = G_GEP %0, %1
+    %3:gpr(s64) = COPY $x2
+    G_STORE %3, %ptr :: (store 8 into %ir.addr)
+...
+---
+name:            strdrox
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0, $x1, $d2
+    ; CHECK-LABEL: name: strdrox
+    ; CHECK: liveins: $x0, $x1, $d2
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[COPY2:%[0-9]+]]:fpr64 = COPY $d2
+    ; CHECK: STRDroX [[COPY2]], [[COPY]], [[COPY1]], 0, 0 :: (store 8 into %ir.addr)
+    %0:gpr(p0) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %ptr:gpr(p0) = G_GEP %0, %1
+    %3:fpr(s64) = COPY $d2
+    G_STORE %3, %ptr :: (store 8 into %ir.addr)
+...
+---
+name:            strwrox
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0, $x1, $w2
+    ; CHECK-LABEL: name: strwrox
+    ; CHECK: liveins: $x0, $x1, $w2
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+    ; CHECK: STRWroX [[COPY2]], [[COPY]], [[COPY1]], 0, 0 :: (store 4 into %ir.addr)
+    %0:gpr(p0) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %ptr:gpr(p0) = G_GEP %0, %1
+    %3:gpr(s32) = COPY $w2
+    G_STORE %3, %ptr :: (store 4 into %ir.addr)
+...
+---
+name:            strsrox
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0, $x1, $s2
+    ; CHECK-LABEL: name: strsrox
+    ; CHECK: liveins: $x0, $x1, $s2
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[COPY2:%[0-9]+]]:fpr32 = COPY $s2
+    ; CHECK: STRSroX [[COPY2]], [[COPY]], [[COPY1]], 0, 0 :: (store 4 into %ir.addr)
+    %0:gpr(p0) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %ptr:gpr(p0) = G_GEP %0, %1
+    %3:fpr(s32) = COPY $s2
+    G_STORE %3, %ptr :: (store 4 into %ir.addr)
+...
+---
+name:            strhrox
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0, $x1, $h0
+    ; CHECK-LABEL: name: strhrox
+    ; CHECK: liveins: $x0, $x1, $h0
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[COPY2:%[0-9]+]]:fpr16 = COPY $h0
+    ; CHECK: STRHroX [[COPY2]], [[COPY]], [[COPY1]], 0, 0 :: (store 2 into %ir.addr)
+    %0:gpr(p0) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %ptr:gpr(p0) = G_GEP %0, %1
+    %3:fpr(s16) = COPY $h0
+    G_STORE %3, %ptr :: (store 2 into %ir.addr)
+...
+---
+name:            strqrox
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0, $x1, $q2
+    ; CHECK-LABEL: name: strqrox
+    ; CHECK: liveins: $x0, $x1, $q2
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[COPY2:%[0-9]+]]:fpr128 = COPY $q2
+    ; CHECK: STRQroX [[COPY2]], [[COPY]], [[COPY1]], 0, 0 :: (store 16 into %ir.addr)
+    %0:gpr(p0) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %ptr:gpr(p0) = G_GEP %0, %1
+    %2:fpr(<2 x s64>) = COPY $q2
+    G_STORE %2, %ptr :: (store 16 into %ir.addr)
+...
+---
+name:            shl
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0, $x1, $x2
+    ; CHECK-LABEL: name: shl
+    ; CHECK: liveins: $x0, $x1, $x2
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1
+    ; CHECK: [[COPY2:%[0-9]+]]:gpr64 = COPY $x2
+    ; CHECK: STRXroX [[COPY2]], [[COPY1]], [[COPY]], 0, 1 :: (store 8 into %ir.addr)
+    %0:gpr(s64) = COPY $x0
+    %1:gpr(s64) = G_CONSTANT i64 3
+    %2:gpr(s64) = G_SHL %0, %1(s64)
+    %3:gpr(p0) = COPY $x1
+    %ptr:gpr(p0) = G_GEP %3, %2
+    %4:gpr(s64) = COPY $x2
+    G_STORE %4, %ptr :: (store 8 into %ir.addr)

llvm/trunk/test/CodeGen/AArch64/arm64-fastisel-gep-promote-before-add.ll

 ; fastisel should not fold add with non-pointer bitwidth
 ; sext(a) + sext(b) != sext(a + b)
-; RUN: llc -mtriple=arm64-apple-darwin %s -O0 -o - | FileCheck %s
+; RUN: llc -fast-isel -mtriple=arm64-apple-darwin %s -O0 -o - | FileCheck %s
 
 define zeroext i8 @gep_promotion(i8* %ptr) nounwind uwtable ssp {
 entry:
   %ptr.addr = alloca i8*, align 8
   %add = add i8 64, 64 ; 0x40 + 0x40
   %0 = load i8*, i8** %ptr.addr, align 8
 
   ; CHECK-LABEL: _gep_promotion:
   ; CHECK: ldrb {{[a-z][0-9]+}}, {{\[[a-z][0-9]+\]}}
   %arrayidx = getelementptr inbounds i8, i8* %0, i8 %add
 
   %1 = load i8, i8* %arrayidx, align 1
   ret i8 %1
 }