[globalisel][legalizerinfo] Add support for legalization based on the MachineMemOperand

[dsanders]

Summary

Currently only the memory size is supported but others can be added as
needed.

narrowScalar for G_LOAD and G_STORE now correctly update the
MachineMemOperand and will refuse to legalize atomics since those need more
careful expansions to maintain atomicity.

[dsanders]

Rebase to trunk

[aemerson]

LGTM.

[dsanders]

Thanks. I'll commit this when I'm back in the office on Thursday

include/llvm/CodeGen/GlobalISel/LegalizerInfo.h

 /// The LegalityQuery object bundles together all the information that's needed
 /// to decide whether a given operation is legal or not.
 /// For efficiency, it doesn't make a copy of Types so care must be taken not
 /// to free it before using the query.
 struct LegalityQuery {
   unsigned Opcode;
   ArrayRef<LLT> Types;
 
+  struct MemDesc {
+    uint64_t Size;
+  };
+
+  /// Operations which require memory can use this to place requirements on the
+  /// memory type for each MMO.
+  ArrayRef<MemDesc> MMODescrs;
+
+  constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types,
+                          const ArrayRef<MemDesc> MMODescrs)
+      : Opcode(Opcode), Types(Types), MMODescrs(MMODescrs) {}
+  constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types)
+      : LegalityQuery(Opcode, Types, {}) {}
+
   raw_ostream &print(raw_ostream &OS) const;
 };
 
 /// The result of a query. It either indicates a final answer of Legal or
 /// Unsupported or describes an action that must be taken to make an operation
 /// more legal.
 struct LegalizeActionStep {
   /// The action to take or the final answer.

 /// True iff the given type index is one of the specified types.
 LegalityPredicate typeInSet(unsigned TypeIdx,
                             std::initializer_list<LLT> TypesInit);
 /// True iff the given types for the given pair of type indexes is one of the
 /// specified type pairs.
 LegalityPredicate
 typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1,
               std::initializer_list<std::pair<LLT, LLT>> TypesInit);
+/// True iff the given types for the given pair of type indexes is one of the
+/// specified type pairs.
+LegalityPredicate typePairAndMemSizeInSet(
+    unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx,
+    std::initializer_list<std::tuple<LLT, LLT, unsigned>> TypesAndMemSizeInit);
 /// True iff the specified type index is a scalar.
 LegalityPredicate isScalar(unsigned TypeIdx);
 /// True iff the specified type index is a scalar that's narrower than the given
 /// size.
 LegalityPredicate narrowerThan(unsigned TypeIdx, unsigned Size);
 /// True iff the specified type index is a scalar that's wider than the given
 /// size.
 LegalityPredicate widerThan(unsigned TypeIdx, unsigned Size);
 /// True iff the specified type index is a scalar whose size is not a power of
 /// 2.
 LegalityPredicate sizeNotPow2(unsigned TypeIdx);
+/// True iff the specified MMO index has a size that is not a power of 2
+LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx);
 /// True iff the specified type index is a vector whose element count is not a
 /// power of 2.
 LegalityPredicate numElementsNotPow2(unsigned TypeIdx);
 } // end namespace LegalityPredicates
 
 namespace LegalizeMutations {
 /// Select this specific type for the given type index.
 LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty);
public:
   LegalizeRuleSet &legalFor(std::initializer_list<LLT> Types) {
     return actionFor(LegalizeAction::Legal, Types);
   }
   /// The instruction is legal when type indexes 0 and 1 is any type pair in the
   /// given list.
   LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
     return actionFor(LegalizeAction::Legal, Types);
   }
+  /// The instruction is legal when type indexes 0 and 1 along with the memory
+  /// size is any type and size tuple in the given list.
+  LegalizeRuleSet &legalForTypesWithMemSize(
+      std::initializer_list<std::tuple<LLT, LLT, unsigned>> TypesAndMemSize) {
+    return legalIf(LegalityPredicates::typePairAndMemSizeInSet(
+        0, 1, /*MMOIdx*/ 0, TypesAndMemSize));
+  }
   /// The instruction is legal when type indexes 0 and 1 are both in the given
   /// list. That is, the type pair is in the cartesian product of the list.
   LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types) {
     return actionForCartesianProduct(LegalizeAction::Legal, Types);
   }
   /// The instruction is legal when type indexes 0 and 1 are both their
   /// respective lists.
   LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0,
public:
 
   /// The instruction is unsupported.
   LegalizeRuleSet &unsupported() {
     return actionIf(LegalizeAction::Unsupported, always);
   }
   LegalizeRuleSet &unsupportedIf(LegalityPredicate Predicate) {
     return actionIf(LegalizeAction::Unsupported, Predicate);
   }
+  LegalizeRuleSet &unsupportedIfMemSizeNotPow2() {
+    return actionIf(LegalizeAction::Unsupported,
+                    LegalityPredicates::memSizeInBytesNotPow2(0));
+  }
 
   LegalizeRuleSet &customIf(LegalityPredicate Predicate) {
     return actionIf(LegalizeAction::Custom, Predicate);
   }
   LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) {
     return actionFor(LegalizeAction::Custom, Types);
   }
   LegalizeRuleSet &customForCartesianProduct(std::initializer_list<LLT> Types) {
public:
   }
 
   /// Limit the number of elements in EltTy vectors to at least MinElements.
   LegalizeRuleSet &clampMinNumElements(unsigned TypeIdx, const LLT &EltTy,
                                        unsigned MinElements) {
     return moreElementsIf(
         [=](const LegalityQuery &Query) {
           LLT VecTy = Query.Types[TypeIdx];
-          return VecTy.getElementType() == EltTy &&
+          return VecTy.isVector() && VecTy.getElementType() == EltTy &&
                  VecTy.getNumElements() < MinElements;
         },
         [=](const LegalityQuery &Query) {
           LLT VecTy = Query.Types[TypeIdx];
           return std::make_pair(
               TypeIdx, LLT::vector(MinElements, VecTy.getScalarSizeInBits()));
         });
   }
   /// Limit the number of elements in EltTy vectors to at most MaxElements.
   LegalizeRuleSet &clampMaxNumElements(unsigned TypeIdx, const LLT &EltTy,
                                        unsigned MaxElements) {
     return fewerElementsIf(
         [=](const LegalityQuery &Query) {
           LLT VecTy = Query.Types[TypeIdx];
-          return VecTy.getElementType() == EltTy &&
+          return VecTy.isVector() && VecTy.getElementType() == EltTy &&
                  VecTy.getNumElements() > MaxElements;
         },
         [=](const LegalityQuery &Query) {
           LLT VecTy = Query.Types[TypeIdx];
           return std::make_pair(
               TypeIdx, LLT::vector(MaxElements, VecTy.getScalarSizeInBits()));
         });
   }

lib/CodeGen/GlobalISel/LegalityPredicates.cpp

LegalityPredicate LegalityPredicates::typePairInSet(
     std::initializer_list<std::pair<LLT, LLT>> TypesInit) {
   SmallVector<std::pair<LLT, LLT>, 4> Types = TypesInit;
   return [=](const LegalityQuery &Query) {
     std::pair<LLT, LLT> Match = {Query.Types[TypeIdx0], Query.Types[TypeIdx1]};
     return std::find(Types.begin(), Types.end(), Match) != Types.end();
   };
 }
 
+LegalityPredicate LegalityPredicates::typePairAndMemSizeInSet(
+    unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx,
+    std::initializer_list<std::tuple<LLT, LLT, unsigned>> TypesAndMemSizeInit) {
+  SmallVector<std::tuple<LLT, LLT, unsigned>, 4> TypesAndMemSize = TypesAndMemSizeInit;
+  return [=](const LegalityQuery &Query) {
+    std::tuple<LLT, LLT, unsigned> Match = {
+        Query.Types[TypeIdx0], Query.Types[TypeIdx1], Query.MMODescrs[MMOIdx].Size};
+    return std::find(TypesAndMemSize.begin(), TypesAndMemSize.end(), Match) !=
+           TypesAndMemSize.end();
+  };
+}
+
 LegalityPredicate LegalityPredicates::isScalar(unsigned TypeIdx) {
   return [=](const LegalityQuery &Query) {
     return Query.Types[TypeIdx].isScalar();
   };
 }
 
 LegalityPredicate LegalityPredicates::narrowerThan(unsigned TypeIdx,
                                                    unsigned Size) {

 
 LegalityPredicate LegalityPredicates::sizeNotPow2(unsigned TypeIdx) {
   return [=](const LegalityQuery &Query) {
     const LLT &QueryTy = Query.Types[TypeIdx];
     return QueryTy.isScalar() && !isPowerOf2_32(QueryTy.getSizeInBits());
   };
 }
 
+LegalityPredicate LegalityPredicates::memSizeInBytesNotPow2(unsigned MMOIdx) {
+  return [=](const LegalityQuery &Query) {
+    return !isPowerOf2_32(Query.MMODescrs[MMOIdx].Size /* In Bytes */);
+  };
+}
+
 LegalityPredicate LegalityPredicates::numElementsNotPow2(unsigned TypeIdx) {
   return [=](const LegalityQuery &Query) {
     const LLT &QueryTy = Query.Types[TypeIdx];
     return QueryTy.isVector() && isPowerOf2_32(QueryTy.getNumElements());
   };
 }

lib/CodeGen/GlobalISel/LegalizerHelper.cpp

LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
                                                               unsigned TypeIdx,
                                                               LLT NarrowTy) {
   // FIXME: Don't know how to handle secondary types yet.
   if (TypeIdx != 0 && MI.getOpcode() != TargetOpcode::G_EXTRACT)
     return UnableToLegalize;
 
   MIRBuilder.setInstr(MI);
 
-  int64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
-  int64_t NarrowSize = NarrowTy.getSizeInBits();
+  uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
+  uint64_t NarrowSize = NarrowTy.getSizeInBits();
 
   switch (MI.getOpcode()) {
   default:
     return UnableToLegalize;
   case TargetOpcode::G_IMPLICIT_DEF: {
     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     // NarrowSize.
     if (SizeOp0 % NarrowSize != 0)
if (SizeOp1 % NarrowSize != 0)
       return UnableToLegalize;
     int NumParts = SizeOp1 / NarrowSize;
 
     SmallVector<unsigned, 2> SrcRegs, DstRegs;
     SmallVector<uint64_t, 2> Indexes;
     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
 
     unsigned OpReg = MI.getOperand(0).getReg();
-    int64_t OpStart = MI.getOperand(2).getImm();
-    int64_t OpSize = MRI.getType(OpReg).getSizeInBits();
+    uint64_t OpStart = MI.getOperand(2).getImm();
+    uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
     for (int i = 0; i < NumParts; ++i) {
       unsigned SrcStart = i * NarrowSize;
 
       if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
         // No part of the extract uses this subregister, ignore it.
         continue;
       } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
         // The entire subregister is extracted, forward the value.
         DstRegs.push_back(SrcRegs[i]);
         continue;
       }
 
       // OpSegStart is where this destination segment would start in OpReg if it
       // extended infinitely in both directions.
-      int64_t ExtractOffset, SegSize;
+      int64_t ExtractOffset;
+      uint64_t SegSize;
       if (OpStart < SrcStart) {
         ExtractOffset = 0;
         SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
       } else {
         ExtractOffset = OpStart - SrcStart;
         SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
       }
 
case TargetOpcode::G_INSERT: {
 
     int NumParts = SizeOp0 / NarrowSize;
 
     SmallVector<unsigned, 2> SrcRegs, DstRegs;
     SmallVector<uint64_t, 2> Indexes;
     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
 
     unsigned OpReg = MI.getOperand(2).getReg();
-    int64_t OpStart = MI.getOperand(3).getImm();
-    int64_t OpSize = MRI.getType(OpReg).getSizeInBits();
+    uint64_t OpStart = MI.getOperand(3).getImm();
+    uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
     for (int i = 0; i < NumParts; ++i) {
       unsigned DstStart = i * NarrowSize;
 
       if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
         // No part of the insert affects this subregister, forward the original.
         DstRegs.push_back(SrcRegs[i]);
         continue;
       } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
         // The entire subregister is defined by this insert, forward the new
         // value.
         DstRegs.push_back(OpReg);
         continue;
       }
 
       // OpSegStart is where this destination segment would start in OpReg if it
       // extended infinitely in both directions.
-      int64_t ExtractOffset, InsertOffset, SegSize;
+      int64_t ExtractOffset, InsertOffset;
+      uint64_t SegSize;
       if (OpStart < DstStart) {
         InsertOffset = 0;
         ExtractOffset = DstStart - OpStart;
         SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
       } else {
         InsertOffset = OpStart - DstStart;
         ExtractOffset = 0;
         SegSize =
case TargetOpcode::G_INSERT: {
     MI.eraseFromParent();
     return Legalized;
   }
   case TargetOpcode::G_LOAD: {
     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     // NarrowSize.
     if (SizeOp0 % NarrowSize != 0)
       return UnableToLegalize;
+
+    const auto &MMO = **MI.memoperands_begin();
+    // This implementation doesn't work for atomics. Give up instead of doing
+    // something invalid.
+    if (MMO.getOrdering() != AtomicOrdering::NotAtomic ||
+        MMO.getFailureOrdering() != AtomicOrdering::NotAtomic)
+      return UnableToLegalize;
+
     int NumParts = SizeOp0 / NarrowSize;
     LLT OffsetTy = LLT::scalar(
         MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
 
     SmallVector<unsigned, 2> DstRegs;
     for (int i = 0; i < NumParts; ++i) {
       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
       unsigned SrcReg = 0;
       unsigned Adjustment = i * NarrowSize / 8;
 
+      MachineMemOperand *SplitMMO = MIRBuilder.getMF().getMachineMemOperand(
+          MMO.getPointerInfo().getWithOffset(Adjustment), MMO.getFlags(),
+          NarrowSize / 8, i == 0 ? MMO.getAlignment() : NarrowSize / 8,
+          MMO.getAAInfo(), MMO.getRanges(), MMO.getSyncScopeID(),
+          MMO.getOrdering(), MMO.getFailureOrdering());
+
       MIRBuilder.materializeGEP(SrcReg, MI.getOperand(1).getReg(), OffsetTy,
                                 Adjustment);
 
-      // TODO: This is conservatively correct, but we probably want to split the
-      // memory operands in the future.
-      MIRBuilder.buildLoad(DstReg, SrcReg, **MI.memoperands_begin());
+      MIRBuilder.buildLoad(DstReg, SrcReg, *SplitMMO);
 
       DstRegs.push_back(DstReg);
     }
     unsigned DstReg = MI.getOperand(0).getReg();
     MIRBuilder.buildMerge(DstReg, DstRegs);
     MI.eraseFromParent();
     return Legalized;
   }
   case TargetOpcode::G_STORE: {
     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     // NarrowSize.
     if (SizeOp0 % NarrowSize != 0)
       return UnableToLegalize;
+
+    const auto &MMO = **MI.memoperands_begin();
+    // This implementation doesn't work for atomics. Give up instead of doing
+    // something invalid.
+    if (MMO.getOrdering() != AtomicOrdering::NotAtomic ||
+        MMO.getFailureOrdering() != AtomicOrdering::NotAtomic)
+      return UnableToLegalize;
+
     int NumParts = SizeOp0 / NarrowSize;
     LLT OffsetTy = LLT::scalar(
         MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
 
     SmallVector<unsigned, 2> SrcRegs;
     extractParts(MI.getOperand(0).getReg(), NarrowTy, NumParts, SrcRegs);
 
     for (int i = 0; i < NumParts; ++i) {
       unsigned DstReg = 0;
       unsigned Adjustment = i * NarrowSize / 8;
 
+      MachineMemOperand *SplitMMO = MIRBuilder.getMF().getMachineMemOperand(
+          MMO.getPointerInfo().getWithOffset(Adjustment), MMO.getFlags(),
+          NarrowSize / 8, i == 0 ? MMO.getAlignment() : NarrowSize / 8,
+          MMO.getAAInfo(), MMO.getRanges(), MMO.getSyncScopeID(),
+          MMO.getOrdering(), MMO.getFailureOrdering());
+
       MIRBuilder.materializeGEP(DstReg, MI.getOperand(1).getReg(), OffsetTy,
                                 Adjustment);
 
-      // TODO: This is conservatively correct, but we probably want to split the
-      // memory operands in the future.
-      MIRBuilder.buildStore(SrcRegs[i], DstReg, **MI.memoperands_begin());
+      MIRBuilder.buildStore(SrcRegs[i], DstReg, *SplitMMO);
     }
     MI.eraseFromParent();
     return Legalized;
   }
   case TargetOpcode::G_CONSTANT: {
     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     // NarrowSize.
     if (SizeOp0 % NarrowSize != 0)

lib/CodeGen/GlobalISel/LegalizerInfo.cpp

 #define DEBUG_TYPE "legalizer-info"
 
 cl::opt<bool> llvm::DisableGISelLegalityCheck(
     "disable-gisel-legality-check",
     cl::desc("Don't verify that MIR is fully legal between GlobalISel passes"),
     cl::Hidden);
 
 raw_ostream &LegalityQuery::print(raw_ostream &OS) const {
-  OS << Opcode << ", {";
+  OS << Opcode << ", Tys={";
   for (const auto &Type : Types) {
     OS << Type << ", ";
   }
+  OS << "}, Opcode=";
+
+  OS << Opcode << ", MMOs={";
+  for (const auto &MMODescr : MMODescrs) {
+    OS << MMODescr.Size << ", ";
+  }
   OS << "}";
+
   return OS;
 }
 
 LegalizeActionStep LegalizeRuleSet::apply(const LegalityQuery &Query) const {
   DEBUG(dbgs() << "Applying legalizer ruleset to: "; Query.print(dbgs());
         dbgs() << "\n");
   if (Rules.empty()) {
     DEBUG(dbgs() << ".. fallback to legacy rules (no rules defined)\n");
for (unsigned i = 0; i < MI.getDesc().getNumOperands(); ++i) {
     if (SeenTypes[TypeIdx])
       continue;
 
     SeenTypes.set(TypeIdx);
 
     LLT Ty = getTypeFromTypeIdx(MI, MRI, i, TypeIdx);
     Types.push_back(Ty);
   }
-  return getAction({MI.getOpcode(), Types});
+
+  SmallVector<LegalityQuery::MemDesc, 2> MemDescrs;
+  for (const auto &MMO : MI.memoperands())
+    MemDescrs.push_back({MMO->getSize() /* in bytes */ * 8});
+
+  return getAction({MI.getOpcode(), Types, MemDescrs});
 }
 
 bool LegalizerInfo::isLegal(const MachineInstr &MI,
                             const MachineRegisterInfo &MRI) const {
   return getAction(MI, MRI).Action == Legal;
 }
 
 bool LegalizerInfo::legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI,

lib/Target/AArch64/AArch64LegalizerInfo.cpp

getActionDefinitionsBuilder(G_EXTRACT)
       })
       .clampScalar(1, s32, s64)
       .widenScalarToNextPow2(1)
       .maxScalarIf(typeInSet(1, {s32}), 0, s16)
       .maxScalarIf(typeInSet(1, {s64}), 0, s32)
       .widenScalarToNextPow2(0);
 
   getActionDefinitionsBuilder({G_LOAD, G_STORE})
-      .legalFor(
-          {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}, {p0, p0}, {v2s32, p0}})
+      .legalForTypesWithMemSize({{s8, p0, 8},
+                                 {s16, p0, 16},
+                                 {s32, p0, 32},
+                                 {s64, p0, 64},
+                                 {p0, p0, 64},
+                                 {v2s32, p0, 64}})
+      // TODO: We could support sum-of-pow2's but the lowering code doesn't know
+      //       how to do that yet.
+      .unsupportedIfMemSizeNotPow2()
       .clampScalar(0, s8, s64)
       .widenScalarToNextPow2(0)
       .clampNumElements(0, v2s32, v2s32);
 
   // Constants
   getActionDefinitionsBuilder(G_CONSTANT)
       .legalFor({p0, s32, s64})
       .clampScalar(0, s32, s64)

test/CodeGen/AArch64/GlobalISel/legalize-extracts.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=aarch64-linux-gnu -O0 -run-pass=legalizer -global-isel %s -o - | FileCheck %s
 
 ---
 name:            test_extracts_1
 body: |
   bb.0:
     liveins: $w0
 
     ; Low part of extraction takes entirity of the low register entirely, so
     ; value stored is forwarded directly from first load.
 
     ; CHECK-LABEL: name: test_extracts_1
     ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x2
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p0) :: (load 16)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p0) :: (load 8, align 16)
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
     ; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s64)
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p0) :: (load 16)
+    ; CHECK: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p0) :: (load 8)
     ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY [[LOAD]](s64)
     ; CHECK: G_STORE [[COPY1]](s64), [[COPY]](p0) :: (store 8)
     ; CHECK: RET_ReallyLR
     %0:_(s64) = COPY $x0
     %1:_(s32) = COPY $w1
     %2:_(p0) = COPY $x2
     %3:_(s128) = G_LOAD %2(p0) :: (load 16)
     %4:_(s64) = G_EXTRACT %3(s128), 0
     G_STORE %4(s64), %2(p0) :: (store 8)
     RET_ReallyLR
 ...
 
 ---
 name:            test_extracts_2
 body: |
   bb.0:
     liveins: $w0
 
       ; Low extraction wipes takes whole low register. High extraction is real.
     ; CHECK-LABEL: name: test_extracts_2
     ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x2
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p0) :: (load 16)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p0) :: (load 8, align 16)
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
     ; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s64)
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p0) :: (load 16)
+    ; CHECK: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p0) :: (load 8)
     ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY [[LOAD]](s64)
     ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[LOAD1]](s64), 0
     ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[EXTRACT]](s32)
     ; CHECK: G_STORE [[COPY1]](s64), [[COPY]](p0) :: (store 8)
     ; CHECK: G_STORE [[COPY2]](s32), [[COPY]](p0) :: (store 4)
     ; CHECK: RET_ReallyLR
     %0:_(s64) = COPY $x0
     %1:_(s32) = COPY $w1

test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir

bb.0.entry:
     %12:_(s64) = G_PTRTOINT %6
     $x0 = COPY %12
 
     ; CHECK: %7:_(<2 x s32>) = G_LOAD %0(p0) :: (load 8 from %ir.addr)
     %7(<2 x s32>) = G_LOAD %0(p0) :: (load 8 from %ir.addr)
     %13:_(s64) = G_BITCAST %7
     $x0 = COPY %13
 
-    ; CHECK: [[LOAD0:%[0-9]+]]:_(s64) = G_LOAD %0(p0) :: (load 16 from %ir.addr)
+    ; CHECK: [[LOAD0:%[0-9]+]]:_(s64) = G_LOAD %0(p0) :: (load 8 from %ir.addr, align 16)
     ; CHECK: [[OFFSET1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
     ; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_GEP %0, [[OFFSET1]](s64)
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p0) :: (load 16 from %ir.addr)
+    ; CHECK: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p0) :: (load 8 from %ir.addr + 8)
     ; CHECK: %8:_(s128) = G_MERGE_VALUES [[LOAD0]](s64), [[LOAD1]](s64)
     %8(s128) = G_LOAD %0(p0) :: (load 16 from %ir.addr)
     %14:_(s64) = G_TRUNC %8
     $x0 = COPY %14
 ...
 
 ---
 name:            test_store
bb.0.entry:
 
     ; CHECK: G_STORE %5(s64), %0(p0) :: (store 8 into %ir.addr)
     %5(s64) = G_PTRTOINT %0(p0)
     G_STORE %5, %0 :: (store 8 into %ir.addr)
 
     ; CHECK: G_STORE %0(p0), %0(p0) :: (store 8 into %ir.addr)
     G_STORE %0(p0), %0(p0) :: (store 8 into %ir.addr)
 
-    ; CHECK: G_STORE %5(s64), %0(p0) :: (store 16 into %ir.addr)
+    ; CHECK: G_STORE %5(s64), %0(p0) :: (store 8 into %ir.addr, align 16)
     ; CHECK: [[OFFSET1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
     ; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_GEP %0, [[OFFSET1]](s64)
-    ; CHECK: G_STORE %6(s64), [[GEP1]](p0) :: (store 16 into %ir.addr)
+    ; CHECK: G_STORE %6(s64), [[GEP1]](p0) :: (store 8 into %ir.addr + 8)
     %6(s64) = G_PTRTOINT %0(p0)
     %7(s128) = G_MERGE_VALUES %5, %6
     G_STORE %7, %0 :: (store 16 into %ir.addr)
 ...

test/CodeGen/ARM/GlobalISel/arm-legalize-vfp4.mir

bb.0:
     ; SOFT: ADJCALLSTACKDOWN
     ; SOFT-ABI-DAG: $r{{[0-1]}} = COPY [[X0]]
     ; SOFT-ABI-DAG: $r{{[0-1]}} = COPY [[X1]]
     ; SOFT-ABI-DAG: $r{{[2-3]}} = COPY [[X0]]
     ; SOFT-ABI-DAG: $r{{[2-3]}} = COPY [[X1]]
     ; SOFT-ABI: [[SP1:%[0-9]+]]:_(p0) = COPY $sp
     ; SOFT-ABI: [[OFF1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; SOFT-ABI: [[FI1:%[0-9]+]]:_(p0) = G_GEP [[SP1]], [[OFF1]](s32)
-    ; SOFT-ABI: G_STORE [[Y0]](s32), [[FI1]](p0){{.*}}store 8 into stack
+
+    ; FIXME: This ought to be align 8 but ARM's call lowering hardcodes it to 0
+    ; SOFT-ABI: G_STORE [[Y0]](s32), [[FI1]](p0){{.*}}store 4 into stack, align 0)
     ; SOFT-ABI: [[OFF2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; SOFT-ABI: [[FI2:%[0-9]+]]:_(p0) = G_GEP [[FI1]], [[OFF2]](s32)
-    ; SOFT-ABI: G_STORE [[Y1]](s32), [[FI2]](p0){{.*}}store 8 into stack
+    ; SOFT-ABI: G_STORE [[Y1]](s32), [[FI2]](p0){{.*}}store 4 into stack + 4)
     ; SOFT-ABI: BL &fma, {{.*}}, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0, implicit-def $r1
     ; SOFT-ABI-DAG: [[R0:%[0-9]+]]:_(s32) = COPY $r0
     ; SOFT-ABI-DAG: [[R1:%[0-9]+]]:_(s32) = COPY $r1
     ; HARD-ABI-DAG: $d0 = COPY [[X]]
     ; HARD-ABI-DAG: $d1 = COPY [[X]]
     ; HARD-ABI-DAG: $d2 = COPY [[Y]]
     ; HARD-ABI: BL &fma, {{.*}}, implicit $d0, implicit $d1, implicit $d2, implicit-def $d0
     ; HARD-ABI: [[R:%[0-9]+]]:_(s64) = COPY $d0

test/CodeGen/X86/GlobalISel/legalize-memop-scalar.mir

bb.1 (%ir-block.0):
     liveins: $rdi
 
     ; X64-LABEL: name: test_memop_s64
     ; X64: [[DEF:%[0-9]+]]:_(p0) = IMPLICIT_DEF
     ; X64: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[DEF]](p0) :: (load 8)
     ; X64: G_STORE [[LOAD]](s64), [[DEF]](p0) :: (store 8)
     ; X32-LABEL: name: test_memop_s64
     ; X32: [[DEF:%[0-9]+]]:_(p0) = IMPLICIT_DEF
-    ; X32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p0) :: (load 8)
+    ; X32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p0) :: (load 4, align 8)
     ; X32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; X32: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[DEF]], [[C]](s32)
-    ; X32: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP]](p0) :: (load 8)
-    ; X32: G_STORE [[LOAD]](s32), [[DEF]](p0) :: (store 8)
+    ; X32: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP]](p0) :: (load 4)
+    ; X32: G_STORE [[LOAD]](s32), [[DEF]](p0) :: (store 4, align 8)
     ; X32: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; X32: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[DEF]], [[C1]](s32)
-    ; X32: G_STORE [[LOAD1]](s32), [[GEP1]](p0) :: (store 8)
+    ; X32: G_STORE [[LOAD1]](s32), [[GEP1]](p0) :: (store 4)
     %0(p0) = IMPLICIT_DEF
     %1(s64) = G_LOAD %0(p0) :: (load 8)
 
     G_STORE %1, %0 :: (store 8)
 
 ...

test/CodeGen/X86/GlobalISel/legalize-undef.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=x86_64-linux-gnu -global-isel -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
 # RUN: llc -mtriple=i386-linux-gnu   -global-isel -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X32
 
 ---
 name:            test_implicit_def
 registers:
 #
 body: |
   bb.0.entry:
     liveins:
     ; X64-LABEL: name: test_implicit_def
     ; X64: [[DEF:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
     ; X64: [[DEF1:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
     ; X64: G_STORE [[DEF1]](s8), [[DEF]](p0) :: (store 1)
     ; X64: [[DEF2:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
-    ; X64: G_STORE [[DEF2]](s8), [[DEF]](p0) :: (store 8)
+    ; X64: G_STORE [[DEF2]](s8), [[DEF]](p0) :: (store 1)
     ; X64: [[DEF3:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
-    ; X64: G_STORE [[DEF3]](s16), [[DEF]](p0) :: (store 16)
+    ; X64: G_STORE [[DEF3]](s16), [[DEF]](p0) :: (store 2)
     ; X64: [[DEF4:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; X64: G_STORE [[DEF4]](s32), [[DEF]](p0) :: (store 32)
+    ; X64: G_STORE [[DEF4]](s32), [[DEF]](p0) :: (store 4)
     ; X64: [[DEF5:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
-    ; X64: G_STORE [[DEF5]](s64), [[DEF]](p0) :: (store 64)
+    ; X64: G_STORE [[DEF5]](s64), [[DEF]](p0) :: (store 8)
     ; X32-LABEL: name: test_implicit_def
     ; X32: [[DEF:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
     ; X32: [[DEF1:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
     ; X32: G_STORE [[DEF1]](s8), [[DEF]](p0) :: (store 1)
     ; X32: [[DEF2:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
-    ; X32: G_STORE [[DEF2]](s8), [[DEF]](p0) :: (store 8)
+    ; X32: G_STORE [[DEF2]](s8), [[DEF]](p0) :: (store 1)
     ; X32: [[DEF3:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
-    ; X32: G_STORE [[DEF3]](s16), [[DEF]](p0) :: (store 16)
+    ; X32: G_STORE [[DEF3]](s16), [[DEF]](p0) :: (store 2)
     ; X32: [[DEF4:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; X32: G_STORE [[DEF4]](s32), [[DEF]](p0) :: (store 32)
+    ; X32: G_STORE [[DEF4]](s32), [[DEF]](p0) :: (store 4)
     ; X32: [[DEF5:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
     ; X32: [[DEF6:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; X32: G_STORE [[DEF5]](s32), [[DEF]](p0) :: (store 64)
+    ; X32: G_STORE [[DEF5]](s32), [[DEF]](p0) :: (store 4, align 8)
     ; X32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; X32: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[DEF]], [[C]](s32)
-    ; X32: G_STORE [[DEF6]](s32), [[GEP]](p0) :: (store 64)
+    ; X32: G_STORE [[DEF6]](s32), [[GEP]](p0) :: (store 4)
     %5:_(p0) = G_IMPLICIT_DEF
     %0:_(s1) = G_IMPLICIT_DEF
     G_STORE %0, %5 ::(store 1)
     %1:_(s8) = G_IMPLICIT_DEF
-    G_STORE %1, %5 ::(store 8)
+    G_STORE %1, %5 ::(store 1)
     %2:_(s16) = G_IMPLICIT_DEF
-    G_STORE %2, %5 ::(store 16)
+    G_STORE %2, %5 ::(store 2)
     %3:_(s32) = G_IMPLICIT_DEF
-    G_STORE %3, %5 ::(store 32)
+    G_STORE %3, %5 ::(store 4)
     %4:_(s64) = G_IMPLICIT_DEF
-    G_STORE %4, %5 ::(store 64)
+    G_STORE %4, %5 ::(store 8)
 ...