AArch64: add FastISel support for arm64_32

[t.p.northover]

Summary

This patch is pretty simple, it just adds the necessary pointer handling to AArch64FastISel.cpp so that it can be used on arm64_32.

[t.p.northover]

Ping.

[Gerolf]

A question and a nit to get it started.

+ Amara, Jessica, Francis.

llvm/lib/Target/AArch64/AArch64FastISel.cpp

1206

Should that check be before the canonicalizations above?

2133

if (Subtarget->isTargetILP32() && ....), then delete the assert. Maybe create a function/lambda? See below.

2210

only if subtarget is ilp32 ...

[Gerolf]

Ups, I removed the original reviewers?! Please add anyone else I missed. I only remember Kristof. Sorry for this nuisance!

-Gerolf

[thegameg]

I don't have much knowledge in this part of the code, but everything looks good to me. Maybe add a simple comment at the top of the file with an overview of what was needed to add support for ILP32 (maybe pointing out that it consists of using ands, extractsubregs, etc.) ?

llvm/lib/CodeGen/SelectionDAG/FastISel.cpp

for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
     EVT VT = RetTys[I];
     MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT);
     unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT);
     for (unsigned i = 0; i != NumRegs; ++i) {
       ISD::InputArg MyFlags;
       MyFlags.VT = RegisterVT;
       MyFlags.ArgVT = VT;
       MyFlags.Used = CLI.IsReturnValueUsed;
+      if (CLI.RetTy->isPointerTy())
+        MyFlags.Flags.setPointer();
       if (CLI.RetSExt)
         MyFlags.Flags.setSExt();
       if (CLI.RetZExt)
         MyFlags.Flags.setZExt();
       if (CLI.IsInReg)
         MyFlags.Flags.setInReg();
       CLI.Ins.push_back(MyFlags);
     }
   }
 
   // Handle all of the outgoing arguments.
   CLI.clearOuts();
   for (auto &Arg : CLI.getArgs()) {
     Type *FinalType = Arg.Ty;
     if (Arg.IsByVal)
       FinalType = cast<PointerType>(Arg.Ty)->getElementType();
     bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
         FinalType, CLI.CallConv, CLI.IsVarArg);
 
     ISD::ArgFlagsTy Flags;
+    if (Arg.Ty->isPointerTy())
+      Flags.setPointer();
     if (Arg.IsZExt)
       Flags.setZExt();
     if (Arg.IsSExt)
       Flags.setSExt();
     if (Arg.IsInReg)
       Flags.setInReg();
     if (Arg.IsSRet)
       Flags.setSRet();

llvm/lib/Target/AArch64/AArch64FastISel.cpp

bool AArch64FastISel::computeCallAddress(const Value *V, Address &Addr) {
   }
 
   return false;
 }
 
 bool AArch64FastISel::isTypeLegal(Type *Ty, MVT &VT) {
   EVT evt = TLI.getValueType(DL, Ty, true);
 
-  if (Subtarget->isTargetILP32() && Ty->isPointerTy())
-    return false;
-
   // Only handle simple types.
   if (evt == MVT::Other || !evt.isSimple())
     return false;
   VT = evt.getSimpleVT();
 
   // This is a legal type, but it's not something we handle in fast-isel.
   if (VT == MVT::f128)
     return false;
bool AArch64FastISel::isValueAvailable(const Value *V) const {
   if (!isa<Instruction>(V))
     return true;
 
   const auto *I = cast<Instruction>(V);
   return FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB;
 }
 
 bool AArch64FastISel::simplifyAddress(Address &Addr, MVT VT) {
-  if (Subtarget->isTargetILP32())
-    return false;
-
   unsigned ScaleFactor = getImplicitScaleFactor(VT);
   if (!ScaleFactor)
     return false;
 
   bool ImmediateOffsetNeedsLowering = false;
   bool RegisterOffsetNeedsLowering = false;
   int64_t Offset = Addr.getOffset();
   if (((Offset < 0) || (Offset & (ScaleFactor - 1))) && !isInt<9>(Offset))
unsigned AArch64FastISel::emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
   unsigned LHSReg = getRegForValue(LHS);
   if (!LHSReg)
     return 0;
   bool LHSIsKill = hasTrivialKill(LHS);
 
   if (NeedExtend)
     LHSReg = emitIntExt(SrcVT, LHSReg, RetVT, IsZExt);
 
+  bool IsILP32Pointer =
+      Subtarget->isTargetILP32() && RHS->getType()->isPointerTy();
+
+  const auto &ExtendResult = [&](unsigned ResultReg) -> unsigned {
+    if (!ResultReg || !IsILP32Pointer)
+      return ResultReg;
+
+    unsigned Result64 = createResultReg(&AArch64::GPR64RegClass);
+    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+            TII.get(TargetOpcode::SUBREG_TO_REG))
+        .addDef(Result64)
+        .addImm(0)
+        .addReg(ResultReg, RegState::Kill)
+        .addImm(AArch64::sub_32);
+    return Result64;
+  };
+
+  if (IsILP32Pointer) {
+    RetVT = MVT::i32;
+    LHSReg =
+        fastEmitInst_extractsubreg(MVT::i32, LHSReg, false, AArch64::sub_32);
+  }
+
+
   unsigned ResultReg = 0;
   if (const auto *C = dyn_cast<ConstantInt>(RHS)) {
     uint64_t Imm = IsZExt ? C->getZExtValue() : C->getSExtValue();
     if (C->isNegative())
       ResultReg = emitAddSub_ri(!UseAdd, RetVT, LHSReg, LHSIsKill, -Imm,
                                 SetFlags, WantResult);
     else
       ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, LHSIsKill, Imm, SetFlags,
                                 WantResult);
   } else if (const auto *C = dyn_cast<Constant>(RHS))
     if (C->isNullValue())
       ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, LHSIsKill, 0, SetFlags,
                                 WantResult);
 
   if (ResultReg)
-    return ResultReg;
+    return ExtendResult(ResultReg);
 
   // Only extend the RHS within the instruction if there is a valid extend type.
   if (ExtendType != AArch64_AM::InvalidShiftExtend && RHS->hasOneUse() &&
       isValueAvailable(RHS)) {
+    assert(!RHS->getType()->isPointerTy() && "ILP32 broken");
     if (const auto *SI = dyn_cast<BinaryOperator>(RHS))
       if (const auto *C = dyn_cast<ConstantInt>(SI->getOperand(1)))
         if ((SI->getOpcode() == Instruction::Shl) && (C->getZExtValue() < 4)) {
           unsigned RHSReg = getRegForValue(SI->getOperand(0));
           if (!RHSReg)
             return 0;
           bool RHSIsKill = hasTrivialKill(SI->getOperand(0));
           return emitAddSub_rx(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg,
if (isMulPowOf2(RHS)) {
       const Value *MulRHS = cast<MulOperator>(RHS)->getOperand(1);
 
       if (const auto *C = dyn_cast<ConstantInt>(MulLHS))
         if (C->getValue().isPowerOf2())
           std::swap(MulLHS, MulRHS);
 
       assert(isa<ConstantInt>(MulRHS) && "Expected a ConstantInt.");
       uint64_t ShiftVal = cast<ConstantInt>(MulRHS)->getValue().logBase2();
+      assert(!RHS->getType()->isPointerTy() && "ILP32 broken");
       unsigned RHSReg = getRegForValue(MulLHS);
       if (!RHSReg)
         return 0;
       bool RHSIsKill = hasTrivialKill(MulLHS);
       ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg,
                                 RHSIsKill, AArch64_AM::LSL, ShiftVal, SetFlags,
                                 WantResult);
       if (ResultReg)
if (const auto *SI = dyn_cast<BinaryOperator>(RHS)) {
         switch (SI->getOpcode()) {
         default: break;
         case Instruction::Shl:  ShiftType = AArch64_AM::LSL; break;
         case Instruction::LShr: ShiftType = AArch64_AM::LSR; break;
         case Instruction::AShr: ShiftType = AArch64_AM::ASR; break;
         }
         uint64_t ShiftVal = C->getZExtValue();
         if (ShiftType != AArch64_AM::InvalidShiftExtend) {
+          assert(!RHS->getType()->isPointerTy() && "ILP32 broken");
           unsigned RHSReg = getRegForValue(SI->getOperand(0));
           if (!RHSReg)
             return 0;
           bool RHSIsKill = hasTrivialKill(SI->getOperand(0));
           ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg,
                                     RHSIsKill, ShiftType, ShiftVal, SetFlags,
                                     WantResult);
           if (ResultReg)
             return ResultReg;
         }
       }
     }
   }
 
   unsigned RHSReg = getRegForValue(RHS);
   if (!RHSReg)
     return 0;
+
+  if (IsILP32Pointer)
+    RHSReg =
+      fastEmitInst_extractsubreg(MVT::i32, RHSReg, false, AArch64::sub_32);
+
   bool RHSIsKill = hasTrivialKill(RHS);
 
   if (NeedExtend)
     RHSReg = emitIntExt(SrcVT, RHSReg, RetVT, IsZExt);
 
-  return emitAddSub_rr(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg, RHSIsKill,
-                       SetFlags, WantResult);
+  return ExtendResult(emitAddSub_rr(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg,
+                                    RHSIsKill, SetFlags, WantResult));
 }
 
 unsigned AArch64FastISel::emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg,
                                         bool LHSIsKill, unsigned RHSReg,
                                         bool RHSIsKill, bool SetFlags,
                                         bool WantResult) {
   assert(LHSReg && RHSReg && "Invalid register number.");
 
bool AArch64FastISel::selectLoad(const Instruction *I) {
   MVT VT;
   // Verify we have a legal type before going any further.  Currently, we handle
   // simple types that will directly fit in a register (i32/f32/i64/f64) or
   // those that can be sign or zero-extended to a basic operation (i1/i8/i16).
   if (!isTypeSupported(I->getType(), VT, /*IsVectorAllowed=*/true) ||
       cast<LoadInst>(I)->isAtomic())
     return false;
 
+  MVT MemVT = TLI.getMemValueType(DL, I->getType()).getSimpleVT();
   const Value *SV = I->getOperand(0);
   if (TLI.supportSwiftError()) {
     // Swifterror values can come from either a function parameter with
     // swifterror attribute or an alloca with swifterror attribute.
     if (const Argument *Arg = dyn_cast<Argument>(SV)) {
       if (Arg->hasSwiftErrorAttr())
         return false;
     }
 
     if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
       if (Alloca->isSwiftError())
         return false;
     }
   }
 
   // See if we can handle this address.
   Address Addr;
   if (!computeAddress(I->getOperand(0), Addr, I->getType()))
     return false;
 
-  // Fold the following sign-/zero-extend into the load instruction.
+  // Fold the following sign-/zero-extend into the load instruction. An ILP32
+  // pointer gets marked for zero-extension at this point.
   bool WantZExt = true;
   MVT RetVT = VT;
   const Value *IntExtVal = nullptr;
   if (I->hasOneUse()) {
     if (const auto *ZE = dyn_cast<ZExtInst>(I->use_begin()->getUser())) {
+      assert(MemVT == RetVT && "unexpected extension of pointer");
       if (isTypeSupported(ZE->getType(), RetVT))
         IntExtVal = ZE;
       else
         RetVT = VT;
     } else if (const auto *SE = dyn_cast<SExtInst>(I->use_begin()->getUser())) {
+      assert(MemVT == RetVT && "unexpected extension of pointer");
       if (isTypeSupported(SE->getType(), RetVT))
         IntExtVal = SE;
       else
         RetVT = VT;
       WantZExt = false;
     }
   }
 
   unsigned ResultReg =
-      emitLoad(VT, RetVT, Addr, WantZExt, createMachineMemOperandFor(I));
+      emitLoad(MemVT, RetVT, Addr, WantZExt, createMachineMemOperandFor(I));
   if (!ResultReg)
     return false;
 
   // There are a few different cases we have to handle, because the load or the
   // sign-/zero-extend might not be selected by FastISel if we fall-back to
   // SelectionDAG. There is also an ordering issue when both instructions are in
   // different basic blocks.
   // 1.) The load instruction is selected by FastISel, but the integer extend
bool AArch64FastISel::emitStoreRelease(MVT VT, unsigned SrcReg,
   default: return false;
   case MVT::i8:  Opc = AArch64::STLRB; break;
   case MVT::i16: Opc = AArch64::STLRH; break;
   case MVT::i32: Opc = AArch64::STLRW; break;
   case MVT::i64: Opc = AArch64::STLRX; break;
   }
 
   const MCInstrDesc &II = TII.get(Opc);
-  SrcReg = constrainOperandRegClass(II, SrcReg, 0);
+  unsigned SubReg = 0;
+  if (VT == MVT::i32 && TRI.getRegSizeInBits(SrcReg, MRI) == 64) {
+    assert(VT == MVT::i32 && TRI.getRegSizeInBits(SrcReg, MRI) == 64 &&
+           Subtarget->isTargetILP32());
+    MRI.constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
+    SubReg = AArch64::sub_32;
+  } else
+    SrcReg = constrainOperandRegClass(II, SrcReg, II.getNumDefs());
+
   AddrReg = constrainOperandRegClass(II, AddrReg, 1);
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
-      .addReg(SrcReg)
-      .addReg(AddrReg)
+      .addUse(SrcReg, 0, SubReg)
+      .addUse(AddrReg)
       .addMemOperand(MMO);
   return true;
 }
 
 bool AArch64FastISel::emitStore(MVT VT, unsigned SrcReg, Address Addr,
                                 MachineMemOperand *MMO) {
   if (!TLI.allowsMisalignedMemoryAccesses(VT))
     return false;
bool AArch64FastISel::emitStore(MVT VT, unsigned SrcReg, Address Addr,
   }
 
   // Storing an i1 requires special handling.
   if (VTIsi1 && SrcReg != AArch64::WZR) {
     unsigned ANDReg = emitAnd_ri(MVT::i32, SrcReg, /*TODO:IsKill=*/false, 1);
     assert(ANDReg && "Unexpected AND instruction emission failure.");
     SrcReg = ANDReg;
   }
-  // Create the base instruction, then add the operands.
+
   const MCInstrDesc &II = TII.get(Opc);
+  unsigned SubReg = 0;
+  if (VT == MVT::i32 && TRI.getRegSizeInBits(SrcReg, MRI) == 64) {
+    MRI.constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
+    SubReg = AArch64::sub_32;
+  } else
     SrcReg = constrainOperandRegClass(II, SrcReg, II.getNumDefs());
+
+  // Create the base instruction, then add the operands.
   MachineInstrBuilder MIB =
-      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).addReg(SrcReg);
+      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
+          .addUse(SrcReg, 0, SubReg);
   addLoadStoreOperands(Addr, MIB, MachineMemOperand::MOStore, ScaleFactor, MMO);
 
   return true;
 }
 
 bool AArch64FastISel::selectStore(const Instruction *I) {
   MVT VT;
   const Value *Op0 = I->getOperand(0);
   // Verify we have a legal type before going any further.  Currently, we handle
   // simple types that will directly fit in a register (i32/f32/i64/f64) or
   // those that can be sign or zero-extended to a basic operation (i1/i8/i16).
   if (!isTypeSupported(Op0->getType(), VT, /*IsVectorAllowed=*/true))
     return false;
 
+  auto *SI = cast<StoreInst>(I);
+  MVT MemVT =
+      TLI.getMemValueType(DL, SI->getOperand(0)->getType()).getSimpleVT();
   const Value *PtrV = I->getOperand(1);
   if (TLI.supportSwiftError()) {
     // Swifterror values can come from either a function parameter with
     // swifterror attribute or an alloca with swifterror attribute.
     if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
       if (Arg->hasSwiftErrorAttr())
         return false;
     }
 
     if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
       if (Alloca->isSwiftError())
         return false;
     }
   }
 
   // Get the value to be stored into a register. Use the zero register directly
   // when possible to avoid an unnecessary copy and a wasted register.
   unsigned SrcReg = 0;
   if (const auto *CI = dyn_cast<ConstantInt>(Op0)) {
     if (CI->isZero())
-      SrcReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
+      SrcReg = (MemVT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
   } else if (const auto *CF = dyn_cast<ConstantFP>(Op0)) {
     if (CF->isZero() && !CF->isNegative()) {
-      VT = MVT::getIntegerVT(VT.getSizeInBits());
-      SrcReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
+      MemVT = MVT::getIntegerVT(VT.getSizeInBits());
+      SrcReg = (MemVT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
     }
   }
 
   if (!SrcReg)
     SrcReg = getRegForValue(Op0);
 
   if (!SrcReg)
     return false;
 
-  auto *SI = cast<StoreInst>(I);
-
   // Try to emit a STLR for seq_cst/release.
   if (SI->isAtomic()) {
     AtomicOrdering Ord = SI->getOrdering();
     // The non-atomic instructions are sufficient for relaxed stores.
     if (isReleaseOrStronger(Ord)) {
       // The STLR addressing mode only supports a base reg; pass that directly.
       unsigned AddrReg = getRegForValue(PtrV);
-      return emitStoreRelease(VT, SrcReg, AddrReg,
+      return emitStoreRelease(MemVT, SrcReg, AddrReg,
                               createMachineMemOperandFor(I));
     }
   }
 
   // See if we can handle this address.
   Address Addr;
   if (!computeAddress(PtrV, Addr, Op0->getType()))
     return false;
 
-  if (!emitStore(VT, SrcReg, Addr, createMachineMemOperandFor(I)))
+  if (!emitStore(MemVT, SrcReg, Addr, createMachineMemOperandFor(I)))
     return false;
   return true;
 }
 
 static AArch64CC::CondCode getCompareCC(CmpInst::Predicate Pred) {
   switch (Pred) {
   case CmpInst::FCMP_ONE:
   case CmpInst::FCMP_UEQ:
bool AArch64FastISel::emitCompareAndBranch(const BranchInst *BI) {
   MVT VT;
   if (!isTypeSupported(LHS->getType(), VT))
     return false;
 
   unsigned BW = VT.getSizeInBits();
   if (BW > 64)
     return false;
 
+  // Signed ILP32 comparisons must be done at 32-bits width because the pointer
+  // is zero-extended to 64-bits.
+  bool IsILP32Pointer = false;
+  if (Subtarget->isTargetILP32() && LHS->getType()->isPointerTy()) {
+    IsILP32Pointer = true;
+    BW = 32;
+  }
+
   MachineBasicBlock *TBB = FuncInfo.MBBMap[BI->getSuccessor(0)];
   MachineBasicBlock *FBB = FuncInfo.MBBMap[BI->getSuccessor(1)];
 
   // Try to take advantage of fallthrough opportunities.
   if (FuncInfo.MBB->isLayoutSuccessor(TBB)) {
     std::swap(TBB, FBB);
     Predicate = CmpInst::getInversePredicate(Predicate);
   }
bool AArch64FastISel::emitCompareAndBranch(const BranchInst *BI) {
   unsigned Opc = OpcTable[IsBitTest][IsCmpNE][Is64Bit];
   const MCInstrDesc &II = TII.get(Opc);
 
   unsigned SrcReg = getRegForValue(LHS);
   if (!SrcReg)
     return false;
   bool SrcIsKill = hasTrivialKill(LHS);
 
-  if (BW == 64 && !Is64Bit)
+  if ((BW == 64 && !Is64Bit) || IsILP32Pointer)
     SrcReg = fastEmitInst_extractsubreg(MVT::i32, SrcReg, SrcIsKill,
                                         AArch64::sub_32);
 
   if ((BW < 32) && !IsBitTest)
     SrcReg = emitIntExt(VT, SrcReg, MVT::i32, /*isZExt=*/true);
 
   // Emit the combined compare and branch instruction.
   SrcReg = constrainOperandRegClass(II, SrcReg,  II.getNumDefs());
bool AArch64FastISel::processCallArgs(CallLoweringInfo &CLI,
   unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown))
     .addImm(NumBytes).addImm(0);
 
   // Process the args.
   for (CCValAssign &VA : ArgLocs) {
     const Value *ArgVal = CLI.OutVals[VA.getValNo()];
     MVT ArgVT = OutVTs[VA.getValNo()];
+    auto ArgFlags = CLI.OutFlags[VA.getValNo()];
 
     unsigned ArgReg = getRegForValue(ArgVal);
     if (!ArgReg)
       return false;
 
     // Handle arg promotion: SExt, ZExt, AExt.
     switch (VA.getLocInfo()) {
     case CCValAssign::Full:
for (CCValAssign &VA : ArgLocs) {
     case CCValAssign::ZExt: {
       MVT DestVT = VA.getLocVT();
       MVT SrcVT = ArgVT;
       ArgReg = emitIntExt(SrcVT, ArgReg, DestVT, /*isZExt=*/true);
       if (!ArgReg)
         return false;
       break;
     }
+    case CCValAssign::Trunc: {
+      assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::i64);
+      ArgVT = MVT::i32;
+      ArgReg =
+          fastEmitInst_extractsubreg(ArgVT, ArgReg, false, AArch64::sub_32);
+      if (!ArgReg)
+        return false;
+      break;
+    }
     default:
       llvm_unreachable("Unknown arg promotion!");
     }
 
     // Now copy/store arg to correct locations.
     if (VA.isRegLoc() && !VA.needsCustom()) {
+      if (Subtarget->isTargetILP32() && ArgFlags.isPointer())
+        ArgReg = emitAnd_ri(MVT::i64, ArgReg, false, 0xffffffff);
+
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
               TII.get(TargetOpcode::COPY), VA.getLocReg()).addReg(ArgReg);
       CLI.OutRegs.push_back(VA.getLocReg());
     } else if (VA.needsCustom()) {
       // FIXME: Handle custom args.
       return false;
     } else {
       assert(VA.isMemLoc() && "Assuming store on stack.");
bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
 
   if (!Callee && !Symbol)
     return false;
 
   // Allow SelectionDAG isel to handle tail calls.
   if (IsTailCall)
     return false;
 
-  // FIXME: we could and should support this, but for now correctness at -O0 is
-  // more important.
-  if (Subtarget->isTargetILP32())
-    return false;
-
   CodeModel::Model CM = TM.getCodeModel();
   // Only support the small-addressing and large code models.
   if (CM != CodeModel::Large && !Subtarget->useSmallAddressing())
     return false;
 
   // FIXME: Add large code model support for ELF.
   if (CM == CodeModel::Large && !Subtarget->isTargetMachO())
     return false;
std::pair<unsigned, bool> AArch64FastISel::getRegForGEPIndex(const Value *Idx) {
   return std::pair<unsigned, bool>(IdxN, IdxNIsKill);
 }
 
 /// This is mostly a copy of the existing FastISel GEP code, but we have to
 /// duplicate it for AArch64, because otherwise we would bail out even for
 /// simple cases. This is because the standard fastEmit functions don't cover
 /// MUL at all and ADD is lowered very inefficientily.
 bool AArch64FastISel::selectGetElementPtr(const Instruction *I) {
-  if (Subtarget->isTargetILP32())
-    return false;
-
   unsigned N = getRegForValue(I->getOperand(0));
   if (!N)
     return false;
   bool NIsKill = hasTrivialKill(I->getOperand(0));
 
   // Keep a running tab of the total offset to coalesce multiple N = N + Offset
   // into a single N = N + TotalOffset.
   uint64_t TotalOffs = 0;
if (auto *StTy = GTI.getStructTypeOrNull()) {
         return false;
     }
   }
   if (TotalOffs) {
     N = emitAdd_ri_(VT, N, NIsKill, TotalOffs);
     if (!N)
       return false;
   }
+
+  if (Subtarget->isTargetILP32() && !cast<GetElementPtrInst>(I)->isInBounds())
+    N = emitAnd_ri(MVT::i64, N, NIsKill, 0xffffffffu);
+
   updateValueMap(I, N);
   return true;
 }
 
 bool AArch64FastISel::selectAtomicCmpXchg(const AtomicCmpXchgInst *I) {
   assert(TM.getOptLevel() == CodeGenOpt::None &&
          "cmpxchg survived AtomicExpand at optlevel > -O0");
 

llvm/test/CodeGen/AArch64/arm64_32-fastisel.ll

-; RUN: llc -mtriple=arm64_32-apple-ios -O0 -fast-isel %s -o - | FileCheck %s
+; RUN: llc -mtriple=arm64_32-apple-ios -O0 -fast-isel -fast-isel-abort=1 %s -o - | FileCheck %s
+
 @var = global i8* null
 
 define void @test_store_release_ptr() {
 ; CHECK-LABEL: test_store_release_ptr
-; CHECK: mov [[ZERO:w[0-9]+]], wzr
-; CHECK: stlr [[ZERO]]
+; CHECK: mov {{w|x}}[[ZERO:[0-9]+]], {{w|x}}zr
+; CHECK: stlr w[[ZERO]]
   store atomic i8* null, i8** @var release, align 4
   br label %next
 
 next:
   ret void
 }
 
 declare [2 x i32] @callee()
 
 define void @test_struct_return(i32* %addr) {
 ; CHECK-LABEL: test_struct_return:
 ; CHECK: bl _callee
 ; CHECK-DAG: lsr [[HI:x[0-9]+]], x0, #32
-; CHECK-DAG: str w0
+; CHECK-DAG: mov {{w[0-9]+}}, w0
   %res = call [2 x i32] @callee()
   %res.0 = extractvalue [2 x i32] %res, 0
   store i32 %res.0, i32* %addr
   %res.1 = extractvalue [2 x i32] %res, 1
   store i32 %res.1, i32* %addr
   ret void
 }
 

 ; CHECK-LABEL: test_ret_ptr_struct:
 ; CHECK: add {{w[0-9]+}}, {{w[0-9]+}}, #1
 
   %sum = add i64 %in, 1
   %res.ptr = inttoptr i64 %sum to i8*
   %res = insertvalue {i8*} undef, i8* %res.ptr, 0
   ret {i8*} %res
 }
+
+
+define void @test_pointer_call(i64 %in) {
+; CHECK-LABEL: test_pointer_call:
+; CHECK: and x0, x0, #0xffffffff
+; CHECK: bl _test_struct_return
+
+  ; Call a random function taking a pointer. Ignore the name.
+  %ptr = inttoptr i64 %in to i32*
+  call void @test_struct_return(i32* %ptr)
+  ret void
+}
+
+define void @test_stack_pointer_call() {
+; CHECK-LABEL: test_stack_pointer_call:
+; CHECK: add x[[VAR:[0-9]+]], sp, #
+; CHECK: mov w[[VAR1:[0-9]+]], w[[VAR]]
+; CHECK-DAG: str w[[VAR1]], [sp]
+; CHECK-DAG: str w[[VAR]], [sp, #4]
+
+  %var = alloca i8
+  call i8* @test_stack_pointer_arg(i64 undef, i64 undef, i64 undef, i64 undef,
+                                   i64 undef, i64 undef, i64 undef, i64 undef,
+                                   i8* %var, i8* %var)
+  ret void
+}
+
+define i8* @test_stack_pointer_arg(i64, i64, i64, i64, i64, i64, i64, i64, i8* %in1, i8* %in2) {
+; CHECK-LABEL: test_stack_pointer_arg:
+; CHECK: ldr [[IN1:w[0-9]+]], [sp]
+; CHECK: mov w[[IN1_TMP:[0-9]+]], [[IN1]]
+; CHECK: and x0, x[[IN1_TMP]], #0xffffffff
+
+  ret i8* %in1
+}
+
+define i8* @test_load_ptr(i8** %addr) {
+; CHECK-LABEL: test_load_ptr:
+; CHECK: ldr [[VAL:w[0-9]+]], [x0, #12]
+; CHECK: mov w[[TMP:[0-9]+]], [[VAL]]
+; CHECK: and x0, x[[TMP]], #0xffffffff
+
+  %elt = getelementptr i8*, i8** %addr, i64 3
+  %val = load i8*, i8** %elt
+  ret i8* %val
+}
+
+define i64 @test_ext_load(i32* %addr) {
+; CHECK-LABEL: test_ext_load:
+; CHECK: ldrsw x0, [x0]
+
+  %val = load i32, i32* %addr
+  %res = sext i32 %val to i64
+  ret i64 %res
+}
+
+define void @test_store_ptr(i8* %in, i8** %addr) {
+; CHECK-LABEL: test_store_ptr:
+; CHECK: str w0, [x1, #12]
+
+  %elt = getelementptr i8*, i8** %addr, i64 3
+  store i8* %in, i8** %elt
+  ret void
+}
+
+define i8* @test_gep(i8* %in) {
+; CHECK-LABEL: test_gep:
+; CHECK: add [[SUM:x[0-9]+]], x0, #12
+; CHECK: and [[MASK:x[0-9]+]], [[SUM]], #0xffffffff
+; CHECK: and x0, [[MASK]], #0xffffffff
+  %res = getelementptr i8, i8* %in, i32 12
+  ret i8* %res
+}
+
+define i8* @test_gep_inbounds(i8* %in) {
+; CHECK-LABEL: test_gep_inbounds:
+; CHECK: add [[SUM:x[0-9]+]], x0, #12
+; CHECK: and x0, [[SUM]], #0xffffffff
+; CHECK-NEXT: ret
+%res = getelementptr inbounds i8, i8* %in, i32 12
+  ret i8* %res
+}
+
+define i1 @test_cmp_bitfield(i8* %in) {
+; CHECK-LABEL: test_cmp_bitfield:
+; CHECK: ubfx {{x[0-9]+}}, x0, #31, #1
+
+  %tst = icmp slt i8* %in, null
+  ret i1 %tst
+}
+
+declare void @foo()
+declare void @bar()
+define void @test_cmp_cbnz(i8* %in) {
+; CHECK-LABEL: test_cmp_cbnz:
+; CHECK: cbnz w0
+
+  %tst = icmp eq i8* %in, null
+  br i1 %tst, label %true, label %false
+
+true:
+  call void @foo()
+  ret void
+
+false:
+  call void @bar()
+  ret void
+}
+
+define void @test_cmp_imm(i8* %in) {
+; CHECK-LABEL: test_cmp_imm:
+; CHECK: subs {{w[0-9]+}}, w0, #41
+; CHECK: b.hi
+
+  %tst = icmp ult i8* %in, inttoptr(i32 42 to i8*)
+  br i1 %tst, label %true, label %false
+
+true:
+  call void @foo()
+  ret void
+
+false:
+  call void @bar()
+  ret void
+}
+
+define void @test_cmp_reg(i8* %lhs, i8* %rhs) {
+; CHECK-LABEL: test_cmp_reg:
+; CHECK: cmp w0, w1
+; CHECK: b.hs
+
+  %tst = icmp ult i8* %lhs, %rhs
+  br i1 %tst, label %true, label %false
+
+true:
+  call void @foo()
+  ret void
+
+false:
+  call void @bar()
+  ret void
+}
+
+define i8* @test_select_ptr(i1 %tst, i8* %lhs, i8* %rhs) {
+; CHECK-LABEL: test_select_ptr:
+; CHECK: tst w0, #0
+; CHECK: csel [[TMP:x[0-9]+]], x1, x2, ne
+; CHECK: and x0, [[TMP]], #0xffffffff
+  %res = select i1 %tst, i8* %lhs, i8* %rhs
+  ret i8* %res
+}

llvm/test/CodeGen/AArch64/arm64_32-null.ll

-; RUN: llc -fast-isel=true  -global-isel=false -O0 -mtriple=arm64_32-apple-ios %s -o - | FileCheck %s
-; RUN: llc -fast-isel=false -global-isel=false -O0 -mtriple=arm64_32-apple-ios %s -o - | FileCheck %s
+; RUN: llc -fast-isel=true  -global-isel=false -O0 -mtriple=arm64_32-apple-ios %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=FAST
+; RUN: llc -fast-isel=false -global-isel=false -O0 -mtriple=arm64_32-apple-ios %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=OPT
 
 define void @test_store(i8** %p) {
 ; CHECK-LABEL: test_store:
-; CHECK: mov [[R1:w[0-9]+]], wzr
-; CHECK: str [[R1]], [x0]
+; CHECK: mov {{x|w}}[[R1:[0-9]+]], {{x|w}}zr
+; CHECK: str w[[R1]], [x0]
 
   store i8* null, i8** %p
   ret void
 }
 
 define void @test_phi(i8** %p) {
 ; CHECK-LABEL: test_phi:
 ; CHECK: mov [[R1:x[0-9]+]], xzr
 ; CHECK: str [[R1]], [sp]
 ; CHECK: b [[BB:LBB[0-9_]+]]
 ; CHECK: [[BB]]:
-; CHECK: ldr x0, [sp]
-; CHECK: str w0, [x{{.*}}]
+; CHECK-OPT: ldr x0, [sp]
+; CHECK-OPT: str w0
+; CHECK-FAST: ldr x[[R2:[0-9]+]], [sp]
+; CHECK-FAST: str [[R2]], [x{{.*}}]
 
 bb0:
   br label %bb1
 bb1:
   %tmp0 = phi i8* [ null, %bb0 ]
   store i8* %tmp0, i8** %p
   ret void
 }

llvm/test/CodeGen/AArch64/arm64_32.ll

 ; CHECK: ret
 
   ret i64 ptrtoint(i32* @var32 to i64)
 }
 
 define i32 @test_global_value() {
 ; CHECK-LABEL: test_global_value:
 ; CHECK: adrp x[[PAGE:[0-9]+]], _var32@PAGE
-; CHECK: ldr w0, [x[[PAGE]], _var32@PAGEOFF]
+; CHECK-OPT: ldr w0, [x[[PAGE]], _var32@PAGEOFF]
+; CHECK-FAST: add x[[VAR32:[0-9]+]], x[[PAGE]], _var32@PAGEOFF
+; CHECK-FAST: ldr w0, [x[[VAR32]]]
   %val = load i32, i32* @var32, align 4
   ret i32 %val
 }
 
 ; Because the addition may wrap, it is not safe to use "ldr w0, [xN, #32]" here.
 define i32 @test_unsafe_indexed_add() {
 ; CHECK-LABEL: test_unsafe_indexed_add:
 ; CHECK: add x[[VAR32:[0-9]+]], {{x[0-9]+}}, _var32@PAGEOFF
 ; CHECK: add w[[ADDR:[0-9]+]], w[[VAR32]], #32
 ; CHECK: ldr w0, [x[[ADDR]]]
   %addr_int = ptrtoint i32* @var32 to i32
   %addr_plus_32 = add i32 %addr_int, 32
   %addr = inttoptr i32 %addr_plus_32 to i32*
   %val = load i32, i32* %addr, align 4
   ret i32 %val
 }
 
 ; Since we've promised there is no unsigned overflow, @var32 must be at least
 ; 32-bytes below 2^32, and we can use the load this time.
 define i32 @test_safe_indexed_add() {
 ; CHECK-LABEL: test_safe_indexed_add:
 ; CHECK: add x[[VAR32:[0-9]+]], {{x[0-9]+}}, _var32@PAGEOFF
-; CHECK: add w[[ADDR:[0-9]+]], w[[VAR32]], #32
-; CHECK: ldr w0, [x[[ADDR]]]
+; CHECK-OPT: add w[[ADDR:[0-9]+]], w[[VAR32]], #32
+; CHECK-OPT: ldr w0, [x[[ADDR]]]
+; CHECK-FAST: ldr w0, [x[[VAR32]], #32]
   %addr_int = ptrtoint i32* @var32 to i64
   %addr_plus_32 = add nuw i64 %addr_int, 32
   %addr = inttoptr i64 %addr_plus_32 to i32*
   %val = load i32, i32* %addr, align 4
   ret i32 %val
 }
 
 define i32 @test_safe_indexed_or(i32 %in) {

 ; CHECK: blr x0
   call void() %func()
   ret void
 }
 
 ; Safe to use the unextended address here
 define void @test_indirect_safe_call(i32* %weird_funcs) {
 ; CHECK-LABEL: test_indirect_safe_call:
-; CHECK: add w[[ADDR32:[0-9]+]], w0, #4
+; CHECK-OPT: add w[[ADDR32:[0-9]+]], w0, #4
 ; CHECK-OPT-NOT: ubfx
+; CHECK-FAST: add [[TMP:x[0-9]+]], x0, #4
+; CHECK-FAST: and x[[ADDR32:[0-9]+]], [[TMP]], #0xffffffff
 ; CHECK: blr x[[ADDR32]]
   %addr = getelementptr i32, i32* %weird_funcs, i32 1
   %func = bitcast i32* %addr to void()*
   call void() %func()
   ret void
 }
 
 declare void @simple()

 declare void @callee([8 x i32]*)
 define void @test_stack_guard() ssp {
 ; CHECK-LABEL: test_stack_guard:
 ; CHECK: adrp x[[GUARD_GOTPAGE:[0-9]+]], ___stack_chk_guard@GOTPAGE
 ; CHECK: ldr w[[GUARD_ADDR:[0-9]+]], [x[[GUARD_GOTPAGE]], ___stack_chk_guard@GOTPAGEOFF]
 ; CHECK: ldr [[GUARD_VAL:w[0-9]+]], [x[[GUARD_ADDR]]]
 ; CHECK: stur [[GUARD_VAL]], [x29, #[[GUARD_OFFSET:-[0-9]+]]]
 
-; CHECK: add x0, sp, #{{[0-9]+}}
+; CHECK-OPT: add x0, sp, #{{[0-9]+}}
+; CHECK-FAST: add [[TMP:x[0-9]+]], sp, #{{[0-9]+}}
+; CHECK-FAST: and x0, [[TMP]], #0xffffffff
 ; CHECK: bl _callee
 
 ; CHECK-OPT: adrp x[[GUARD_GOTPAGE:[0-9]+]], ___stack_chk_guard@GOTPAGE
 ; CHECK-OPT: ldr w[[GUARD_ADDR:[0-9]+]], [x[[GUARD_GOTPAGE]], ___stack_chk_guard@GOTPAGEOFF]
 ; CHECK-OPT: ldr [[GUARD_VAL:w[0-9]+]], [x[[GUARD_ADDR]]]
 ; CHECK-OPT: ldur [[NEW_VAL:w[0-9]+]], [x29, #[[GUARD_OFFSET]]]
 ; CHECK-OPT: cmp [[GUARD_VAL]], [[NEW_VAL]]
 ; CHECK-OPT: b.ne [[FAIL:LBB[0-9]+_[0-9]+]]
 
 ; CHECK-OPT: [[FAIL]]:
 ; CHECK-OPT-NEXT: bl ___stack_chk_fail
   %arr = alloca [8 x i32]
   call void @callee([8 x i32]* %arr)
   ret void
 }
 
 declare i32 @__gxx_personality_v0(...)
 declare void @eat_landingpad_args(i32, i8*, i32)
 @_ZTI8Whatever = external global i8
 define void @test_landingpad_marshalling() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 ; CHECK-LABEL: test_landingpad_marshalling:
 ; CHECK-OPT: mov x2, x1
 ; CHECK-OPT: mov x1, x0
+; CHECK-FAST: mov x2, x1
+; CHECK-FAST: and x1, x0, #0xffffffff
 ; CHECK: bl _eat_landingpad_args
   invoke void @callee([8 x i32]* undef) to label %done unwind label %lpad
 
 lpad:                                             ; preds = %entry
   %exc = landingpad { i8*, i32 }
           catch i8* @_ZTI8Whatever
   %pointer = extractvalue { i8*, i32 } %exc, 0
   %selector = extractvalue { i8*, i32 } %exc, 1

 next:
   %val = alloca [8 x i32]
   call void @callee([8 x i32]* %val)
   ret void
 }
 
 define void @test_asm_memory(i32* %base.addr) {
 ; CHECK-LABEL: test_asm_memory:
-; CHECK: add w[[ADDR:[0-9]+]], w0, #4
+; CHECK-OPT: add w[[ADDR:[0-9]+]], w0, #4
+; CHECK-FAST: add [[TMP:x[0-9]+]], x0, #4
+; CHECK-FAST: and x[[ADDR:[0-9]+]], [[TMP]], #0xffffffff
 ; CHECK: str wzr, [x[[ADDR]]
   %addr = getelementptr i32, i32* %base.addr, i32 1
   call void asm sideeffect "str wzr, $0", "*m"(i32* %addr)
   ret void
 }
 
 define void @test_unsafe_asm_memory(i64 %val) {
 ; CHECK-LABEL: test_unsafe_asm_memory:

 }
 
 define { [18 x i8] }* @test_gep_nonpow2({ [18 x i8] }* %a0, i32 %a1) {
 ; CHECK-LABEL: test_gep_nonpow2:
 ; CHECK-OPT:      mov w[[SIZE:[0-9]+]], #18
 ; CHECK-OPT-NEXT: smaddl x0, w1, w[[SIZE]], x0
 ; CHECK-OPT-NEXT: ret
 
-; CHECK-FAST:      mov w[[SIZE:[0-9]+]], #18
-; CHECK-FAST-NEXT: smaddl [[TMP:x[0-9]+]], w1, w[[SIZE]], x0
+; CHECK-FAST:      sxtw [[EXT:x[0-9]+]], w1
+; CHECK-FAST-NEXT: mov [[SIZE:x[0-9]+]], #18
+; CHECK-FAST-NEXT: madd [[TMP:x[0-9]+]], [[EXT]], [[SIZE]], x0
 ; CHECK-FAST-NEXT: and x0, [[TMP]], #0xffffffff
 ; CHECK-FAST-NEXT: ret
   %tmp0 = getelementptr inbounds { [18 x i8] }, { [18 x i8] }* %a0, i32 %a1
   ret { [18 x i8] }* %tmp0
 }
 
 define void @test_bzero(i64 %in)  {
 ; CHECK-LABEL: test_bzero:

llvm/test/CodeGen/AArch64/swifterror.ll

 ; CHECK-APPLE: strb [[CODE]], [{{.*}}[[ID]]]
 ; CHECK-APPLE: bl {{.*}}free
 
 ; CHECK-O0-LABEL: caller:
 ; CHECK-O0: mov x21
 ; CHECK-O0: bl {{.*}}foo
 ; CHECK-O0: mov [[ID:x[0-9]+]], x21
 ; CHECK-O0-AARCH64: cbnz x21
-; CHECK-O0-ARM64_32: cmp x21, #0
+; CHECK-O0-ARM64_32: mov [[TMP:w[0-9]+]], w21
+; CHECK-O0-ARM64_32: cbnz [[TMP]]
 entry:
   %error_ptr_ref = alloca swifterror %swift_error*
   store %swift_error* null, %swift_error** %error_ptr_ref
   %call = call float @foo(%swift_error** swifterror %error_ptr_ref)
   %error_from_foo = load %swift_error*, %swift_error** %error_ptr_ref
   %had_error_from_foo = icmp ne %swift_error* %error_from_foo, null
   %tmp = bitcast %swift_error* %error_from_foo to i8*
   br i1 %had_error_from_foo, label %handler, label %cont

 ; CHECK-APPLE: mov x0, x21
 ; CHECK-APPLE: bl {{.*}}free
 
 ; CHECK-O0-LABEL: caller2:
 ; CHECK-O0: mov x21
 ; CHECK-O0: bl {{.*}}foo
 ; CHECK-O0: mov [[ID:x[0-9]+]], x21
 ; CHECK-O0-AARCH64: cbnz x21
-; CHECK-O0-ARM64_32: cmp x21, #0
+; CHECK-O0-ARM64_32: mov [[TMP:w[0-9]+]], w21
+; CHECK-O0-ARM64_32: cbnz [[TMP]]
 entry:
   %error_ptr_ref = alloca swifterror %swift_error*
   br label %bb_loop
 bb_loop:
   store %swift_error* null, %swift_error** %error_ptr_ref
   %call = call float @foo(%swift_error** swifterror %error_ptr_ref)
   %error_from_foo = load %swift_error*, %swift_error** %error_ptr_ref
   %had_error_from_foo = icmp ne %swift_error* %error_from_foo, null

 
 ; CHECK-O0-LABEL: caller3:
 ; spill x0
 ; CHECK-O0: str x0
 ; CHECK-O0: mov x21
 ; CHECK-O0: bl {{.*}}foo_sret
 ; CHECK-O0: mov [[ID2:x[0-9]+]], x21
 ; CHECK-O0-AARCH64: cbnz x21
-; CHECK-O0-ARM64_32: cmp x21, #0
+; CHECK-O0-ARM64_32: mov [[TMP:w[0-9]+]], w21
+; CHECK-O0-ARM64_32: cbnz [[TMP]]
 ; Access part of the error object and save it to error_ref
 ; reload from stack
 ; CHECK-O0: ldrb [[CODE:w[0-9]+]]
 ; CHECK-O0: ldr [[ID:x[0-9]+]]
 ; CHECK-O0: strb [[CODE]], [{{.*}}[[ID]]]
 ; CHECK-O0: bl {{.*}}free
 entry:
   %s = alloca %struct.S, align 8