GlobalISel: Use LLT in memory legality queries

[arsenm]

Summary

This enables proper lowering of non-byte sized loads. We still aren't
faithfully preserving memory types everywhere, so the legality checks
still only consider the size.

[arsenm]

Re-enable some commented out tests that work now

[aemerson]

Why did the AArch64 tests change?

llvm/lib/Target/ARM/ARMInstructionSelector.cpp

1099–1116

?

[arsenm]

In D104000#2850901, @aemerson wrote:

Why did the AArch64 tests change?

Because this wasn't properly handling non-byte accesses. The current DAG behavior is to zero the high bits on store, and insert AssertZext on load. When the legalizer handles this, it makes the special case s1 handling in the selector dead (the selector won't even see an s1 memory type)

[aemerson]

Ok, LGTM.

[arsenm]

28f2f66200c5719a637d857387922c91d8b12c34

llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h

 /// to decide whether a given operation is legal or not.
 /// For efficiency, it doesn't make a copy of Types so care must be taken not
 /// to free it before using the query.
 struct LegalityQuery {
   unsigned Opcode;
   ArrayRef<LLT> Types;
 
   struct MemDesc {
-    uint64_t SizeInBits;
+    LLT MemoryTy;
     uint64_t AlignInBits;
     AtomicOrdering Ordering;
   };
 
   /// Operations which require memory can use this to place requirements on the
   /// memory type for each MMO.
   ArrayRef<MemDesc> MMODescrs;
 

 using LegalityPredicate = std::function<bool (const LegalityQuery &)>;
 using LegalizeMutation =
     std::function<std::pair<unsigned, LLT>(const LegalityQuery &)>;
 
 namespace LegalityPredicates {
 struct TypePairAndMemDesc {
   LLT Type0;
   LLT Type1;
-  uint64_t MemSize;
+  LLT MemTy;
   uint64_t Align;
 
   bool operator==(const TypePairAndMemDesc &Other) const {
     return Type0 == Other.Type0 && Type1 == Other.Type1 &&
-           Align == Other.Align &&
-           MemSize == Other.MemSize;
+           Align == Other.Align && MemTy == Other.MemTy;
   }
 
   /// \returns true if this memory access is legal with for the access described
   /// by \p Other (The alignment is sufficient for the size and result type).
   bool isCompatible(const TypePairAndMemDesc &Other) const {
     return Type0 == Other.Type0 && Type1 == Other.Type1 &&
            Align >= Other.Align &&
-           MemSize == Other.MemSize;
+           // FIXME: This perhaps should be stricter, but the current legality
+           // rules are written only considering the size.
+           MemTy.getSizeInBits() == Other.MemTy.getSizeInBits();
   }
 };
 
 /// True iff P0 and P1 are true.
 template<typename Predicate>
 Predicate all(Predicate P0, Predicate P1) {
   return [=](const LegalityQuery &Query) {
     return P0(Query) && P1(Query);

llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

if (UseMI.getOpcode() == TargetOpcode::G_SEXT ||
         (UseMI.getOpcode() == TargetOpcode::G_ANYEXT)) {
       const auto &MMO = **MI.memoperands_begin();
       // For atomics, only form anyextending loads.
       if (MMO.isAtomic() && UseMI.getOpcode() != TargetOpcode::G_ANYEXT)
         continue;
       // Check for legality.
       if (LI) {
         LegalityQuery::MemDesc MMDesc;
-        MMDesc.SizeInBits = MMO.getSizeInBits();
+        MMDesc.MemoryTy = MMO.getMemoryType();
         MMDesc.AlignInBits = MMO.getAlign().value() * 8;
         MMDesc.Ordering = MMO.getOrdering();
         LLT UseTy = MRI.getType(UseMI.getOperand(0).getReg());
         LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
         if (LI->getAction({MI.getOpcode(), {UseTy, SrcTy}, {MMDesc}}).Action !=
             LegalizeActions::Legal)
           continue;
       }
if (ZeroOffsetIdx == MemOffset2Idx.end() ||
       ZeroOffsetIdx->second != LowestIdx)
     return false;
 
   // We wil reuse the pointer from the load which ends up at byte offset 0. It
   // may not use index 0.
   Register Ptr = LowestIdxLoad->getOperand(1).getReg();
   const MachineMemOperand &MMO = **LowestIdxLoad->memoperands_begin();
   LegalityQuery::MemDesc MMDesc;
-  MMDesc.SizeInBits = WideMemSizeInBits;
+  MMDesc.MemoryTy = Ty;
   MMDesc.AlignInBits = MMO.getAlign().value() * 8;
   MMDesc.Ordering = MMO.getOrdering();
   if (!isLegalOrBeforeLegalizer(
           {TargetOpcode::G_LOAD, {Ty, MRI.getType(Ptr)}, {MMDesc}}))
     return false;
   auto PtrInfo = MMO.getPointerInfo();
   auto *NewMMO = MF.getMachineMemOperand(&MMO, PtrInfo, WideMemSizeInBits / 8);
 

llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp

 }
 
 LegalityPredicate LegalityPredicates::typePairAndMemDescInSet(
     unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx,
     std::initializer_list<TypePairAndMemDesc> TypesAndMemDescInit) {
   SmallVector<TypePairAndMemDesc, 4> TypesAndMemDesc = TypesAndMemDescInit;
   return [=](const LegalityQuery &Query) {
     TypePairAndMemDesc Match = {Query.Types[TypeIdx0], Query.Types[TypeIdx1],
-                                Query.MMODescrs[MMOIdx].SizeInBits,
+                                Query.MMODescrs[MMOIdx].MemoryTy,
                                 Query.MMODescrs[MMOIdx].AlignInBits};
     return llvm::any_of(TypesAndMemDesc,
                         [=](const TypePairAndMemDesc &Entry) -> bool {
                           return Match.isCompatible(Entry);
                         });
   };
 }
 
LegalityPredicate LegalityPredicates::sameSize(unsigned TypeIdx0,
   return [=](const LegalityQuery &Query) {
     return Query.Types[TypeIdx0].getSizeInBits() ==
            Query.Types[TypeIdx1].getSizeInBits();
   };
 }
 
 LegalityPredicate LegalityPredicates::memSizeInBytesNotPow2(unsigned MMOIdx) {
   return [=](const LegalityQuery &Query) {
-    return !isPowerOf2_32(Query.MMODescrs[MMOIdx].SizeInBits / 8);
+    return !isPowerOf2_32(Query.MMODescrs[MMOIdx].MemoryTy.getSizeInBytes());
   };
 }
 
 LegalityPredicate LegalityPredicates::numElementsNotPow2(unsigned TypeIdx) {
   return [=](const LegalityQuery &Query) {
     const LLT QueryTy = Query.Types[TypeIdx];
     return QueryTy.isVector() && !isPowerOf2_32(QueryTy.getNumElements());
   };
 }
 
 LegalityPredicate LegalityPredicates::atomicOrderingAtLeastOrStrongerThan(
     unsigned MMOIdx, AtomicOrdering Ordering) {
   return [=](const LegalityQuery &Query) {
     return isAtLeastOrStrongerThan(Query.MMODescrs[MMOIdx].Ordering, Ordering);
   };
 }

llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp

raw_ostream &LegalityQuery::print(raw_ostream &OS) const {
   OS << Opcode << ", Tys={";
   for (const auto &Type : Types) {
     OS << Type << ", ";
   }
   OS << "}, Opcode=";
 
   OS << Opcode << ", MMOs={";
   for (const auto &MMODescr : MMODescrs) {
-    OS << MMODescr.SizeInBits << ", ";
+    OS << MMODescr.MemoryTy << ", ";
   }
   OS << "}";
 
   return OS;
 }
 
 #ifndef NDEBUG
 // Make sure the rule won't (trivially) loop forever.
for (unsigned i = 0; i < MI.getDesc().getNumOperands(); ++i) {
     SeenTypes.set(TypeIdx);
 
     LLT Ty = getTypeFromTypeIdx(MI, MRI, i, TypeIdx);
     Types.push_back(Ty);
   }
 
   SmallVector<LegalityQuery::MemDesc, 2> MemDescrs;
   for (const auto &MMO : MI.memoperands())
-    MemDescrs.push_back({8 * MMO->getSize() /* in bits */,
-                         8 * MMO->getAlign().value(), MMO->getOrdering()});
+    MemDescrs.push_back({MMO->getMemoryType(), 8 * MMO->getAlign().value(),
+                         MMO->getOrdering()});
 
   return getAction({MI.getOpcode(), Types, MemDescrs});
 }
 
 bool LegalizerInfo::isLegal(const MachineInstr &MI,
                             const MachineRegisterInfo &MRI) const {
   return getAction(MI, MRI).Action == Legal;
 }

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

getActionDefinitionsBuilder(G_EXTRACT)
       .clampScalar(1, s32, s128)
       .widenScalarToNextPow2(1)
       .maxScalarIf(typeInSet(1, {s32}), 0, s16)
       .maxScalarIf(typeInSet(1, {s64}), 0, s32)
       .widenScalarToNextPow2(0);
 
   getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
       .lowerIf(atomicOrderingAtLeastOrStrongerThan(0, AtomicOrdering::Unordered))
-      .legalForTypesWithMemDesc({{s32, p0, 8, 8},
-                                 {s32, p0, 16, 8},
-                                 {s32, p0, 32, 8},
-                                 {s64, p0, 8, 2},
-                                 {s64, p0, 16, 2},
-                                 {s64, p0, 32, 4},
-                                 {s64, p0, 64, 8},
-                                 {p0, p0, 64, 8},
-                                 {v2s32, p0, 64, 8}})
+      .legalForTypesWithMemDesc({{s32, p0, s8, 8},
+                                 {s32, p0, s16, 8},
+                                 {s32, p0, s32, 8},
+                                 {s64, p0, s8, 2},
+                                 {s64, p0, s16, 2},
+                                 {s64, p0, s32, 4},
+                                 {s64, p0, s64, 8},
+                                 {p0, p0, s64, 8},
+                                 {v2s32, p0, s64, 8}})
       .clampScalar(0, s32, s64)
       .widenScalarToNextPow2(0)
       // TODO: We could support sum-of-pow2's but the lowering code doesn't know
       //       how to do that yet.
       .unsupportedIfMemSizeNotPow2()
       // Lower anything left over into G_*EXT and G_LOAD
       .lower();
 
   auto IsPtrVecPred = [=](const LegalityQuery &Query) {
     const LLT &ValTy = Query.Types[0];
     if (!ValTy.isVector())
       return false;
     const LLT EltTy = ValTy.getElementType();
     return EltTy.isPointer() && EltTy.getAddressSpace() == 0;
   };
 
   getActionDefinitionsBuilder(G_LOAD)
-      .legalForTypesWithMemDesc({{s8, p0, 8, 8},
-                                 {s16, p0, 16, 8},
-                                 {s32, p0, 32, 8},
-                                 {s64, p0, 64, 8},
-                                 {p0, p0, 64, 8},
-                                 {s128, p0, 128, 8},
-                                 {v8s8, p0, 64, 8},
-                                 {v16s8, p0, 128, 8},
-                                 {v4s16, p0, 64, 8},
-                                 {v8s16, p0, 128, 8},
-                                 {v2s32, p0, 64, 8},
-                                 {v4s32, p0, 128, 8},
-                                 {v2s64, p0, 128, 8}})
+      .legalForTypesWithMemDesc({{s8, p0, s8, 8},
+                                 {s16, p0, s16, 8},
+                                 {s32, p0, s32, 8},
+                                 {s64, p0, s64, 8},
+                                 {p0, p0, s64, 8},
+                                 {s128, p0, s128, 8},
+                                 {v8s8, p0, s64, 8},
+                                 {v16s8, p0, s128, 8},
+                                 {v4s16, p0, s64, 8},
+                                 {v8s16, p0, s128, 8},
+                                 {v2s32, p0, s64, 8},
+                                 {v4s32, p0, s128, 8},
+                                 {v2s64, p0, s128, 8}})
       // These extends are also legal
-      .legalForTypesWithMemDesc({{s32, p0, 8, 8}, {s32, p0, 16, 8}})
+      .legalForTypesWithMemDesc({{s32, p0, s8, 8}, {s32, p0, s16, 8}})
       .clampScalar(0, s8, s64)
       .lowerIfMemSizeNotPow2()
       .widenScalarToNextPow2(0)
       .narrowScalarIf([=](const LegalityQuery &Query) {
         // Clamp extending load results to 32-bits.
         return Query.Types[0].isScalar() &&
-               Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits &&
-               Query.Types[0].getSizeInBits() > 32;
+          Query.Types[0] != Query.MMODescrs[0].MemoryTy &&
         },
         changeTo(0, s32))
       // Lower any any-extending loads left into G_ANYEXT and G_LOAD
       .lowerIf([=](const LegalityQuery &Query) {
-        return Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits;
+        return Query.Types[0] != Query.MMODescrs[0].MemoryTy;
       })
       .clampMaxNumElements(0, s8, 16)
       .clampMaxNumElements(0, s16, 8)
       .clampMaxNumElements(0, s32, 4)
       .clampMaxNumElements(0, s64, 2)
       .customIf(IsPtrVecPred);
 
   getActionDefinitionsBuilder(G_STORE)
-      .legalForTypesWithMemDesc({{s8, p0, 8, 8},
-                                 {s16, p0, 8, 8}, // truncstorei8 from s16
-                                 {s32, p0, 8, 8}, // truncstorei8 from s32
-                                 {s64, p0, 8, 8}, // truncstorei8 from s64
-                                 {s16, p0, 16, 8},
-                                 {s32, p0, 16, 8}, // truncstorei16 from s32
-                                 {s64, p0, 16, 8}, // truncstorei16 from s64
-                                 {s32, p0, 8, 8},
-                                 {s32, p0, 16, 8},
-                                 {s32, p0, 32, 8},
-                                 {s64, p0, 64, 8},
-                                 {s64, p0, 32, 8}, // truncstorei32 from s64
-                                 {p0, p0, 64, 8},
-                                 {s128, p0, 128, 8},
-                                 {v16s8, p0, 128, 8},
-                                 {v8s8, p0, 64, 8},
-                                 {v4s16, p0, 64, 8},
-                                 {v8s16, p0, 128, 8},
-                                 {v2s32, p0, 64, 8},
-                                 {v4s32, p0, 128, 8},
-                                 {v2s64, p0, 128, 8}})
+      .legalForTypesWithMemDesc({{s8, p0, s8, 8},
+                                 {s16, p0, s8, 8}, // truncstorei8 from s16
+                                 {s32, p0, s8, 8}, // truncstorei8 from s32
+                                 {s64, p0, s8, 8}, // truncstorei8 from s64
+                                 {s16, p0, s16, 8},
+                                 {s32, p0, s16, 8}, // truncstorei16 from s32
+                                 {s64, p0, s16, 8}, // truncstorei16 from s64
+                                 {s32, p0, s8, 8},
+                                 {s32, p0, s16, 8},
+                                 {s32, p0, s32, 8},
+                                 {s64, p0, s64, 8},
+                                 {s64, p0, s32, 8}, // truncstorei32 from s64
+                                 {p0, p0, s64, 8},
+                                 {s128, p0, s128, 8},
+                                 {v16s8, p0, s128, 8},
+                                 {v8s8, p0, s64, 8},
+                                 {v4s16, p0, s64, 8},
+                                 {v8s16, p0, s128, 8},
+                                 {v2s32, p0, s64, 8},
+                                 {v4s32, p0, s128, 8},
+                                 {v2s64, p0, s128, 8}})
       .clampScalar(0, s8, s64)
       .lowerIf([=](const LegalityQuery &Query) {
         return Query.Types[0].isScalar() &&
-               Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits;
+               Query.Types[0] != Query.MMODescrs[0].MemoryTy;
       })
       // Maximum: sN * k = 128
       .clampMaxNumElements(0, s8, 16)
       .clampMaxNumElements(0, s16, 8)
       .clampMaxNumElements(0, s32, 4)
       .clampMaxNumElements(0, s64, 2)
       .lowerIfMemSizeNotPow2()
       .customIf(IsPtrVecPred);

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

 }
 
 // If we have a truncating store or an extending load with a data size larger
 // than 32-bits, we need to reduce to a 32-bit type.
 static LegalityPredicate isWideScalarExtLoadTruncStore(unsigned TypeIdx) {
   return [=](const LegalityQuery &Query) {
     const LLT Ty = Query.Types[TypeIdx];
     return !Ty.isVector() && Ty.getSizeInBits() > 32 &&
-           Query.MMODescrs[0].SizeInBits < Ty.getSizeInBits();
+           Query.MMODescrs[0].MemoryTy.getSizeInBits() < Ty.getSizeInBits();
   };
 }
 
 // TODO: Should load to s16 be legal? Most loads extend to 32-bits, but we
 // handle some operations by just promoting the register during
 // selection. There are also d16 loads on GFX9+ which preserve the high bits.
 static unsigned maxSizeForAddrSpace(const GCNSubtarget &ST, unsigned AS,
                                     bool IsLoad) {

 static bool isLoadStoreSizeLegal(const GCNSubtarget &ST,
                                  const LegalityQuery &Query) {
   const LLT Ty = Query.Types[0];
 
   // Handle G_LOAD, G_ZEXTLOAD, G_SEXTLOAD
   const bool IsLoad = Query.Opcode != AMDGPU::G_STORE;
 
   unsigned RegSize = Ty.getSizeInBits();
-  unsigned MemSize = Query.MMODescrs[0].SizeInBits;
+  unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
   unsigned AlignBits = Query.MMODescrs[0].AlignInBits;
   unsigned AS = Query.Types[1].getAddressSpace();
 
   // All of these need to be custom lowered to cast the pointer operand.
   if (AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
     return false;
 
   // Do not handle extending vector loads.
static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query) {
   const LLT Ty = Query.Types[0];
   return isRegisterType(Ty) && isLoadStoreSizeLegal(ST, Query) &&
          !loadStoreBitcastWorkaround(Ty);
 }
 
 /// Return true if a load or store of the type should be lowered with a bitcast
 /// to a different type.
 static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty,
-                                       const unsigned MemSizeInBits) {
+                                       const LLT MemTy) {
+  const unsigned MemSizeInBits = MemTy.getSizeInBits();
   const unsigned Size = Ty.getSizeInBits();
     if (Size != MemSizeInBits)
       return Size <= 32 && Ty.isVector();
 
   if (loadStoreBitcastWorkaround(Ty) && isRegisterType(Ty))
     return true;
-  return Ty.isVector() && (Size <= 32 || isRegisterSize(Size)) &&
+
+  // Don't try to handle bitcasting vector ext loads for now.
+  return Ty.isVector() && (!MemTy.isVector() || MemTy == Ty) &&
+         (Size <= 32 || isRegisterSize(Size)) &&
          !isRegisterVectorElementType(Ty.getElementType());
 }
 
 /// Return true if we should legalize a load by widening an odd sized memory
 /// access up to the alignment. Note this case when the memory access itself
 /// changes, not the size of the result register.
-static bool shouldWidenLoad(const GCNSubtarget &ST, unsigned SizeInBits,
+static bool shouldWidenLoad(const GCNSubtarget &ST, LLT MemoryTy,
                             unsigned AlignInBits, unsigned AddrSpace,
                             unsigned Opcode) {
+  unsigned SizeInBits = MemoryTy.getSizeInBits();
   // We don't want to widen cases that are naturally legal.
   if (isPowerOf2_32(SizeInBits))
     return false;
 
   // If we have 96-bit memory operations, we shouldn't touch them. Note we may
   // end up widening these for a scalar load during RegBankSelect, since there
   // aren't 96-bit scalar loads.
   if (SizeInBits == 96 && ST.hasDwordx3LoadStores())
return TLI->allowsMisalignedMemoryAccessesImpl(
          Fast;
 }
 
 static bool shouldWidenLoad(const GCNSubtarget &ST, const LegalityQuery &Query,
                             unsigned Opcode) {
   if (Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic)
     return false;
 
-  return shouldWidenLoad(ST, Query.MMODescrs[0].SizeInBits,
+  return shouldWidenLoad(ST, Query.MMODescrs[0].MemoryTy,
                          Query.MMODescrs[0].AlignInBits,
                          Query.Types[1].getAddressSpace(), Opcode);
 }
 
 AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
                                          const GCNTargetMachine &TM)
   :  ST(ST_) {
   using namespace TargetOpcode;
getActionDefinitionsBuilder(G_ADDRSPACE_CAST)
     .scalarize(0)
     .custom();
 
   const auto needToSplitMemOp = [=](const LegalityQuery &Query,
                                     bool IsLoad) -> bool {
     const LLT DstTy = Query.Types[0];
 
     // Split vector extloads.
-    unsigned MemSize = Query.MMODescrs[0].SizeInBits;
+    unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
     unsigned AlignBits = Query.MMODescrs[0].AlignInBits;
 
     if (MemSize < DstTy.getSizeInBits())
       MemSize = std::max(MemSize, AlignBits);
 
     if (DstTy.isVector() && DstTy.getSizeInBits() > MemSize)
       return true;
 
AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
   // TODO: Unsupported flat for SI.
 
   for (unsigned Op : {G_LOAD, G_STORE}) {
     const bool IsStore = Op == G_STORE;
 
     auto &Actions = getActionDefinitionsBuilder(Op);
     // Explicitly list some common cases.
     // TODO: Does this help compile time at all?
-    Actions.legalForTypesWithMemDesc({{S32, GlobalPtr, 32, GlobalAlign32},
-                                      {V2S32, GlobalPtr, 64, GlobalAlign32},
-                                      {V4S32, GlobalPtr, 128, GlobalAlign32},
-                                      {S64, GlobalPtr, 64, GlobalAlign32},
-                                      {V2S64, GlobalPtr, 128, GlobalAlign32},
-                                      {V2S16, GlobalPtr, 32, GlobalAlign32},
-                                      {S32, GlobalPtr, 8, GlobalAlign8},
-                                      {S32, GlobalPtr, 16, GlobalAlign16},
-
-                                      {S32, LocalPtr, 32, 32},
-                                      {S64, LocalPtr, 64, 32},
-                                      {V2S32, LocalPtr, 64, 32},
-                                      {S32, LocalPtr, 8, 8},
-                                      {S32, LocalPtr, 16, 16},
-                                      {V2S16, LocalPtr, 32, 32},
-
-                                      {S32, PrivatePtr, 32, 32},
-                                      {S32, PrivatePtr, 8, 8},
-                                      {S32, PrivatePtr, 16, 16},
-                                      {V2S16, PrivatePtr, 32, 32},
-
-                                      {S32, ConstantPtr, 32, GlobalAlign32},
-                                      {V2S32, ConstantPtr, 64, GlobalAlign32},
-                                      {V4S32, ConstantPtr, 128, GlobalAlign32},
-                                      {S64, ConstantPtr, 64, GlobalAlign32},
-                                      {V2S32, ConstantPtr, 32, GlobalAlign32}});
+    Actions.legalForTypesWithMemDesc({{S32, GlobalPtr, S32, GlobalAlign32},
+                                      {V2S32, GlobalPtr, V2S32, GlobalAlign32},
+                                      {V4S32, GlobalPtr, V4S32, GlobalAlign32},
+                                      {S64, GlobalPtr, S64, GlobalAlign32},
+                                      {V2S64, GlobalPtr, V2S64, GlobalAlign32},
+                                      {V2S16, GlobalPtr, V2S16, GlobalAlign32},
+                                      {S32, GlobalPtr, S8, GlobalAlign8},
+                                      {S32, GlobalPtr, S16, GlobalAlign16},
+
+                                      {S32, LocalPtr, S32, 32},
+                                      {S64, LocalPtr, S64, 32},
+                                      {V2S32, LocalPtr, V2S32, 32},
+                                      {S32, LocalPtr, S8, 8},
+                                      {S32, LocalPtr, S16, 16},
+                                      {V2S16, LocalPtr, S32, 32},
+
+                                      {S32, PrivatePtr, S32, 32},
+                                      {S32, PrivatePtr, S8, 8},
+                                      {S32, PrivatePtr, S16, 16},
+                                      {V2S16, PrivatePtr, S32, 32},
+
+                                      {S32, ConstantPtr, S32, GlobalAlign32},
+                                      {V2S32, ConstantPtr, V2S32, GlobalAlign32},
+                                      {V4S32, ConstantPtr, V4S32, GlobalAlign32},
+                                      {S64, ConstantPtr, S64, GlobalAlign32},
+                                      {V2S32, ConstantPtr, V2S32, GlobalAlign32}});
     Actions.legalIf(
       [=](const LegalityQuery &Query) -> bool {
         return isLoadStoreLegal(ST, Query);
       });
 
     // Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
     // 64-bits.
     //
     // TODO: Should generalize bitcast action into coerce, which will also cover
     // inserting addrspacecasts.
     Actions.customIf(typeIs(1, Constant32Ptr));
 
     // Turn any illegal element vectors into something easier to deal
     // with. These will ultimately produce 32-bit scalar shifts to extract the
     // parts anyway.
     //
     // For odd 16-bit element vectors, prefer to split those into pieces with
     // 16-bit vector parts.
     Actions.bitcastIf(
       [=](const LegalityQuery &Query) -> bool {
         return shouldBitcastLoadStoreType(ST, Query.Types[0],
-                                          Query.MMODescrs[0].SizeInBits);
+                                          Query.MMODescrs[0].MemoryTy);
       }, bitcastToRegisterType(0));
 
     if (!IsStore) {
       // Widen suitably aligned loads by loading extra bytes. The standard
       // legalization actions can't properly express widening memory operands.
       Actions.customIf([=](const LegalityQuery &Query) -> bool {
         return shouldWidenLoad(ST, Query, G_LOAD);
       });
     }
 
     // FIXME: load/store narrowing should be moved to lower action
     Actions
         .narrowScalarIf(
             [=](const LegalityQuery &Query) -> bool {
               return !Query.Types[0].isVector() &&
                      needToSplitMemOp(Query, Op == G_LOAD);
             },
             [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
               const LLT DstTy = Query.Types[0];
               const LLT PtrTy = Query.Types[1];
 
               const unsigned DstSize = DstTy.getSizeInBits();
-              unsigned MemSize = Query.MMODescrs[0].SizeInBits;
+              unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
 
               // Split extloads.
               if (DstSize > MemSize)
                 return std::make_pair(0, LLT::scalar(MemSize));
 
               if (!isPowerOf2_32(DstSize)) {
                 // We're probably decomposing an odd sized store. Try to split
                 // to the widest type. TODO: Account for alignment. As-is it
Actions
                                                      PtrTy.getAddressSpace(),
                                                      Op == G_LOAD);
 
               // FIXME: Handle widened to power of 2 results better. This ends
               // up scalarizing.
               // FIXME: 3 element stores scalarized on SI
 
               // Split if it's too large for the address space.
-              if (Query.MMODescrs[0].SizeInBits > MaxSize) {
+              unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
+              if (MemSize > MaxSize) {
                 unsigned NumElts = DstTy.getNumElements();
                 unsigned EltSize = EltTy.getSizeInBits();
 
                 if (MaxSize % EltSize == 0) {
                   return std::make_pair(
                     0, LLT::scalarOrVector(MaxSize / EltSize, EltTy));
                 }
 
-                unsigned NumPieces = Query.MMODescrs[0].SizeInBits / MaxSize;
+                unsigned NumPieces = MemSize / MaxSize;
 
                 // FIXME: Refine when odd breakdowns handled
                 // The scalars will need to be re-legalized.
                 if (NumPieces == 1 || NumPieces >= NumElts ||
                     NumElts % NumPieces != 0)
                   return std::make_pair(0, EltTy);
 
                 return std::make_pair(0,
                                       LLT::vector(NumElts / NumPieces, EltTy));
               }
 
               // FIXME: We could probably handle weird extending loads better.
-              unsigned MemSize = Query.MMODescrs[0].SizeInBits;
               if (DstTy.getSizeInBits() > MemSize)
                 return std::make_pair(0, EltTy);
 
               unsigned EltSize = EltTy.getSizeInBits();
               unsigned DstSize = DstTy.getSizeInBits();
               if (!isPowerOf2_32(DstSize)) {
                 // We're probably decomposing an odd sized store. Try to split
                 // to the widest type. TODO: Account for alignment. As-is it
for (unsigned Op : {G_LOAD, G_STORE}) {
     .minScalar(0, S32)
     .narrowScalarIf(isWideScalarExtLoadTruncStore(0), changeTo(0, S32))
     .widenScalarToNextPow2(0)
     .moreElementsIf(vectorSmallerThan(0, 32), moreEltsToNext32Bit(0))
     .lower();
   }
 
   auto &ExtLoads = getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
-                       .legalForTypesWithMemDesc({{S32, GlobalPtr, 8, 8},
-                                                  {S32, GlobalPtr, 16, 2 * 8},
-                                                  {S32, LocalPtr, 8, 8},
-                                                  {S32, LocalPtr, 16, 16},
-                                                  {S32, PrivatePtr, 8, 8},
-                                                  {S32, PrivatePtr, 16, 16},
-                                                  {S32, ConstantPtr, 8, 8},
-                                                  {S32, ConstantPtr, 16, 2 * 8}})
+                       .legalForTypesWithMemDesc({{S32, GlobalPtr, S8, 8},
+                                                  {S32, GlobalPtr, S16, 2 * 8},
+                                                  {S32, LocalPtr, S8, 8},
+                                                  {S32, LocalPtr, S16, 16},
+                                                  {S32, PrivatePtr, S8, 8},
+                                                  {S32, PrivatePtr, S16, 16},
+                                                  {S32, ConstantPtr, S8, 8},
+                                                  {S32, ConstantPtr, S16, 2 * 8}})
                        .legalIf(
                          [=](const LegalityQuery &Query) -> bool {
                            return isLoadStoreLegal(ST, Query);
                          });
 
   if (ST.hasFlatAddressSpace()) {
     ExtLoads.legalForTypesWithMemDesc(
-        {{S32, FlatPtr, 8, 8}, {S32, FlatPtr, 16, 16}});
+        {{S32, FlatPtr, S8, 8}, {S32, FlatPtr, S16, 16}});
   }
 
   // Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
   // 64-bits.
   //
   // TODO: Should generalize bitcast action into coerce, which will also cover
   // inserting addrspacecasts.
   ExtLoads.customIf(typeIs(1, Constant32Ptr));
bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper,
   if (MI.getOpcode() != AMDGPU::G_LOAD)
     return false;
 
   Register ValReg = MI.getOperand(0).getReg();
   LLT ValTy = MRI.getType(ValReg);
 
   MachineMemOperand *MMO = *MI.memoperands_begin();
   const unsigned ValSize = ValTy.getSizeInBits();
-  const unsigned MemSize = 8 * MMO->getSize();
+  const LLT MemTy = MMO->getMemoryType();
   const Align MemAlign = MMO->getAlign();
+  const unsigned MemSize = MemTy.getSizeInBits();
   const unsigned AlignInBits = 8 * MemAlign.value();
 
   // Widen non-power-of-2 loads to the alignment if needed
-  if (shouldWidenLoad(ST, MemSize, AlignInBits, AddrSpace, MI.getOpcode())) {
+  if (shouldWidenLoad(ST, MemTy, AlignInBits, AddrSpace, MI.getOpcode())) {
     const unsigned WideMemSize = PowerOf2Ceil(MemSize);
 
     // This was already the correct extending load result type, so just adjust
     // the memory type.
     if (WideMemSize == ValSize) {
       MachineFunction &MF = B.getMF();
 
       MachineMemOperand *WideMMO =
bool AMDGPULegalizerInfo::legalizeSBufferLoad(
 
   Register Dst = MI.getOperand(0).getReg();
   LLT Ty = B.getMRI()->getType(Dst);
   unsigned Size = Ty.getSizeInBits();
   MachineFunction &MF = B.getMF();
 
   Observer.changingInstr(MI);
 
-  if (shouldBitcastLoadStoreType(ST, Ty, Size)) {
+  if (shouldBitcastLoadStoreType(ST, Ty, LLT::scalar(Size))) {
     Ty = getBitcastRegisterType(Ty);
     Helper.bitcastDst(MI, Ty, 0);
     Dst = MI.getOperand(0).getReg();
     B.setInsertPt(B.getMBB(), MI);
   }
 
   // FIXME: We don't really need this intermediate instruction. The intrinsic
   // should be fixed to have a memory operand. Since it's readnone, we're not

llvm/lib/Target/ARM/ARMInstructionSelector.cpp

case G_LOAD: {
 
     assert((ValSize != 64 || STI.hasVFP2Base()) &&
            "Don't know how to load/store 64-bit value without VFP");
 
     const auto NewOpc = selectLoadStoreOpCode(I.getOpcode(), RegBank, ValSize);
     if (NewOpc == G_LOAD || NewOpc == G_STORE)
       return false;
 
-    if (ValSize == 1 && NewOpc == Opcodes.STORE8) {
-      // Before storing a 1-bit value, make sure to clear out any unneeded bits.
-      Register OriginalValue = I.getOperand(0).getReg();
-
-      Register ValueToStore = MRI.createVirtualRegister(&ARM::GPRRegClass);
-      I.getOperand(0).setReg(ValueToStore);
-
-      auto InsertBefore = I.getIterator();
-      auto AndI = BuildMI(MBB, InsertBefore, I.getDebugLoc(), TII.get(Opcodes.AND))
-        .addDef(ValueToStore)
-        .addUse(OriginalValue)
-        .addImm(1)
-        .add(predOps(ARMCC::AL))
-        .add(condCodeOp());
-      if (!constrainSelectedInstRegOperands(*AndI, TII, TRI, RBI))
-        return false;
-    }
-
     I.setDesc(TII.get(NewOpc));
 
     if (NewOpc == ARM::LDRH || NewOpc == ARM::STRH)
       // LDRH has a funny addressing mode (there's already a FIXME for it).
       MIB.addReg(0);
     MIB.addImm(0).add(predOps(ARMCC::AL));
     break;
   }

llvm/lib/Target/ARM/ARMLegalizerInfo.cpp

ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) {
 
   getActionDefinitionsBuilder(G_SELECT)
       .legalForCartesianProduct({s32, p0}, {s1})
       .minScalar(0, s32);
 
   // We're keeping these builders around because we'll want to add support for
   // floating point to them.
   auto &LoadStoreBuilder = getActionDefinitionsBuilder({G_LOAD, G_STORE})
-                               .legalForTypesWithMemDesc({{s1, p0, 8, 8},
-                                                          {s8, p0, 8, 8},
-                                                          {s16, p0, 16, 8},
-                                                          {s32, p0, 32, 8},
-                                                          {p0, p0, 32, 8}})
+                               .legalForTypesWithMemDesc({{s8, p0, s8, 8},
+                                                          {s16, p0, s16, 8},
+                                                          {s32, p0, s32, 8},
+                                                          {p0, p0, p0, 8}})
                                .unsupportedIfMemSizeNotPow2();
 
   getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0});
   getActionDefinitionsBuilder(G_GLOBAL_VALUE).legalFor({p0});
 
   auto &PhiBuilder =
       getActionDefinitionsBuilder(G_PHI)
           .legalFor({s32, p0})
           .minScalar(0, s32);
 
   getActionDefinitionsBuilder(G_PTR_ADD)
       .legalFor({{p0, s32}})
       .minScalar(1, s32);
 
   getActionDefinitionsBuilder(G_BRCOND).legalFor({s1});
 
   if (!ST.useSoftFloat() && ST.hasVFP2Base()) {
     getActionDefinitionsBuilder(
         {G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FCONSTANT, G_FNEG})
         .legalFor({s32, s64});
 
     LoadStoreBuilder
-        .legalForTypesWithMemDesc({{s64, p0, 64, 32}})
+        .legalForTypesWithMemDesc({{s64, p0, s64, 32}})
         .maxScalar(0, s32);
     PhiBuilder.legalFor({s64});
 
     getActionDefinitionsBuilder(G_FCMP).legalForCartesianProduct({s1},
                                                                  {s32, s64});
 
     getActionDefinitionsBuilder(G_MERGE_VALUES).legalFor({{s64, s32}});
     getActionDefinitionsBuilder(G_UNMERGE_VALUES).legalFor({{s32, s64}});
if (!ST.useSoftFloat() && ST.hasVFP2Base()) {
     getActionDefinitionsBuilder(G_FPTRUNC).libcallFor({{s32, s64}});
 
     getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
         .libcallForCartesianProduct({s32}, {s32, s64});
     getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
         .libcallForCartesianProduct({s32, s64}, {s32});
   }
 
+  // Just expand whatever loads and stores are left.
+  LoadStoreBuilder.lower();
+
   if (!ST.useSoftFloat() && ST.hasVFP4Base())
     getActionDefinitionsBuilder(G_FMA).legalFor({s32, s64});
   else
     getActionDefinitionsBuilder(G_FMA).libcallFor({s32, s64});
 
   getActionDefinitionsBuilder({G_FREM, G_FPOW}).libcallFor({s32, s64});
 
   if (ST.hasV5TOps()) {

llvm/lib/Target/Mips/MipsLegalizerInfo.cpp

static bool isUnalignedMemmoryAccess(uint64_t MemSize, uint64_t AlignInBits) {
   if (MemSize > AlignInBits)
     return true;
   return false;
 }
 
 static bool
 CheckTy0Ty1MemSizeAlign(const LegalityQuery &Query,
                         std::initializer_list<TypesAndMemOps> SupportedValues) {
-  unsigned QueryMemSize = Query.MMODescrs[0].SizeInBits;
+  unsigned QueryMemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
 
   // Non power of two memory access is never legal.
   if (!isPowerOf2_64(QueryMemSize))
     return false;
 
   for (auto &Val : SupportedValues) {
     if (Val.ValTy != Query.Types[0])
       continue;
static bool CheckTyN(unsigned N, const LegalityQuery &Query,
                      std::initializer_list<LLT> SupportedValues) {
   return llvm::is_contained(SupportedValues, Query.Types[N]);
 }
 
 MipsLegalizerInfo::MipsLegalizerInfo(const MipsSubtarget &ST) {
   using namespace TargetOpcode;
 
   const LLT s1 = LLT::scalar(1);
+  const LLT s8 = LLT::scalar(8);
+  const LLT s16 = LLT::scalar(16);
   const LLT s32 = LLT::scalar(32);
   const LLT s64 = LLT::scalar(64);
   const LLT v16s8 = LLT::vector(16, 8);
   const LLT v8s16 = LLT::vector(8, 16);
   const LLT v4s32 = LLT::vector(4, 32);
   const LLT v2s64 = LLT::vector(2, 64);
   const LLT p0 = LLT::pointer(0, 32);
 
getActionDefinitionsBuilder({G_LOAD, G_STORE})
       // - non-power-of-2 MemSizes
       // - unaligned 2 or 8 byte MemSizes for MIPS32r5 and older
       .customIf([=, &ST](const LegalityQuery &Query) {
         if (!Query.Types[0].isScalar() || Query.Types[1] != p0 ||
             Query.Types[0] == s1)
           return false;
 
         unsigned Size = Query.Types[0].getSizeInBits();
-        unsigned QueryMemSize = Query.MMODescrs[0].SizeInBits;
+        unsigned QueryMemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
         assert(QueryMemSize <= Size && "Scalar can't hold MemSize");
 
         if (Size > 64 || QueryMemSize > 64)
           return false;
 
-        if (!isPowerOf2_64(Query.MMODescrs[0].SizeInBits))
+        if (!isPowerOf2_64(Query.MMODescrs[0].MemoryTy.getSizeInBits()))
           return true;
 
         if (!ST.systemSupportsUnalignedAccess() &&
             isUnalignedMemmoryAccess(QueryMemSize,
                                      Query.MMODescrs[0].AlignInBits)) {
           assert(QueryMemSize != 32 && "4 byte load and store are legal");
           return true;
         }
 
         return false;
       })
-      .minScalar(0, s32);
+      .minScalar(0, s32)
+      .lower();
 
   getActionDefinitionsBuilder(G_IMPLICIT_DEF)
       .legalFor({s32, s64});
 
   getActionDefinitionsBuilder(G_UNMERGE_VALUES)
      .legalFor({{s32, s64}});
 
   getActionDefinitionsBuilder(G_MERGE_VALUES)
      .legalFor({{s64, s32}});
 
   getActionDefinitionsBuilder({G_ZEXTLOAD, G_SEXTLOAD})
-      .legalForTypesWithMemDesc({{s32, p0, 8, 8},
-                                 {s32, p0, 16, 8}})
+      .legalForTypesWithMemDesc({{s32, p0, s8, 8},
+                                 {s32, p0, s16, 8}})
       .clampScalar(0, s32, s32);
 
   getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT})
       .legalIf([](const LegalityQuery &Query) { return false; })
       .maxScalar(0, s32);
 
   getActionDefinitionsBuilder(G_TRUNC)
       .legalIf([](const LegalityQuery &Query) { return false; })

llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -O0 -march=aarch64 -run-pass=legalizer -global-isel-abort=1 %s -o - | FileCheck %s
 ---
 name:            test_load
 body: |
   bb.0:
     liveins: $x0
 
     ; CHECK-LABEL: name: test_load
     ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[COPY]](p0) :: (load (s1))
-    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LOAD]](s8)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[COPY]](p0) :: (load (s8))
+    ; CHECK: [[ASSERT_ZEXT:%[0-9]+]]:_(s8) = G_ASSERT_ZEXT [[LOAD]], 1
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASSERT_ZEXT]](s8)
     ; CHECK: $w0 = COPY [[ANYEXT]](s32)
     ; CHECK: [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[COPY]](p0) :: (load (s8))
     ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LOAD1]](s8)
     ; CHECK: $w0 = COPY [[ANYEXT1]](s32)
     ; CHECK: [[LOAD2:%[0-9]+]]:_(s16) = G_LOAD [[COPY]](p0) :: (load (s16))
     ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[LOAD2]](s16)
     ; CHECK: $w0 = COPY [[ANYEXT2]](s32)
     ; CHECK: $w0 = COPY [[ANYEXT1]](s32)

 name:            test_store
 body: |
   bb.0:
     liveins: $x0, $w1
 
     ; CHECK-LABEL: name: test_store
     ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
-    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
     ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[AND]](s32)
-    ; CHECK: G_STORE [[TRUNC]](s8), [[COPY]](p0) :: (store (s1))
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[AND1]](s32)
+    ; CHECK: G_STORE [[TRUNC]](s8), [[COPY]](p0) :: (store (s8))
     ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
     ; CHECK: G_STORE [[TRUNC1]](s8), [[COPY]](p0) :: (store (s8))
     ; CHECK: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; CHECK: G_STORE [[TRUNC2]](s16), [[COPY]](p0) :: (store (s16))
     ; CHECK: G_STORE [[COPY1]](s32), [[COPY]](p0) :: (store (s32))
     ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p0)
     ; CHECK: G_STORE [[PTRTOINT]](s64), [[COPY]](p0) :: (store (s64))
     ; CHECK: G_STORE [[COPY]](p0), [[COPY]](p0) :: (store (p0))

llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-trunc.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -march=aarch64 -run-pass=legalizer %s -o - -verify-machineinstrs | FileCheck %s
 ---
 name:            test_load_trunc
 stack:
   - { id: 0, type: default, offset: 0, size: 2,
       alignment: 2, stack-id: default, callee-saved-register: '', callee-saved-restored: true,
       debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
 body: |
   bb.0.entry:
 
     ; CHECK-LABEL: name: test_load_trunc
     ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX]](p0) :: (load (s10))
-    ; CHECK: [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[LOAD]](s16)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX]](p0) :: (load (s16))
+    ; CHECK: [[ASSERT_ZEXT:%[0-9]+]]:_(s16) = G_ASSERT_ZEXT [[LOAD]], 10
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[ASSERT_ZEXT]](s16)
     ; CHECK: RET_ReallyLR implicit [[TRUNC]](s1)
     %0:_(p0) = G_FRAME_INDEX %stack.0
     %1:_(s10) = G_LOAD %0(p0) :: (load (s10))
     %2:_(s1) = G_TRUNC %1(s10)
     RET_ReallyLR implicit %2(s1)
 ...

llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=bonaire -O0 -run-pass=legalizer -global-isel-abort=0  %s -o - | FileCheck -check-prefix=CI %s
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=VI %s
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=GFX9 %s
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=GFX9 %s
 
 ---
 name: test_load_constant_s1_align1
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; CI-LABEL: name: test_load_constant_s1_align1
     ; CI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s1), addrspace 4)
+    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 4)
     ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; CI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CI: $vgpr0 = COPY [[AND]](s32)
     ; VI-LABEL: name: test_load_constant_s1_align1
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s1), addrspace 4)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 4)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; VI: $vgpr0 = COPY [[AND]](s32)
     ; GFX9-LABEL: name: test_load_constant_s1_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s1), addrspace 4)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 4)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; GFX9: $vgpr0 = COPY [[AND]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s1) = G_LOAD %0 :: (load (s1), align 1, addrspace 4)
     %2:_(s32) = G_ZEXT %1
     $vgpr0 = COPY %2
 ...
 
 ---
 name: test_load_constant_s2_align1
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; CI-LABEL: name: test_load_constant_s2_align1
     ; CI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load (s2), addrspace 4)
+    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load (s8), addrspace 4)
     ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; CI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CI: $vgpr0 = COPY [[AND]](s32)
     ; VI-LABEL: name: test_load_constant_s2_align1
     ; VI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load (s2), addrspace 4)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load (s8), addrspace 4)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; VI: $vgpr0 = COPY [[AND]](s32)
     ; GFX9-LABEL: name: test_load_constant_s2_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load (s2), addrspace 4)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p4) :: (load (s8), addrspace 4)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; GFX9: $vgpr0 = COPY [[AND]](s32)
     %0:_(p4) = COPY $vgpr0_vgpr1
     %1:_(s2) = G_LOAD %0 :: (load (s2), align 1, addrspace 4)
     %2:_(s32) = G_ZEXT %1
     $vgpr0 = COPY %2

llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=bonaire -O0 -run-pass=legalizer -global-isel-abort=0  %s -o - | FileCheck -check-prefix=CI %s
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=VI %s
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=GFX9 %s
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=GFX9 %s
 
 ---
 name: test_load_flat_s1_align1
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; CI-LABEL: name: test_load_flat_s1_align1
     ; CI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s1))
+    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s8))
     ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; CI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CI: $vgpr0 = COPY [[AND]](s32)
     ; VI-LABEL: name: test_load_flat_s1_align1
     ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s1))
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s8))
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; VI: $vgpr0 = COPY [[AND]](s32)
     ; GFX9-LABEL: name: test_load_flat_s1_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s1))
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s8))
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; GFX9: $vgpr0 = COPY [[AND]](s32)
     %0:_(p0) = COPY $vgpr0_vgpr1
     %1:_(s1) = G_LOAD %0 :: (load (s1), align 1, addrspace 0)
     %2:_(s32) = G_ZEXT %1
     $vgpr0 = COPY %2
 ...
 
 ---
 name: test_load_flat_s2_align1
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; CI-LABEL: name: test_load_flat_s2_align1
     ; CI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s2))
+    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s8))
     ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; CI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CI: $vgpr0 = COPY [[AND]](s32)
     ; VI-LABEL: name: test_load_flat_s2_align1
     ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s2))
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s8))
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; VI: $vgpr0 = COPY [[AND]](s32)
     ; GFX9-LABEL: name: test_load_flat_s2_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s2))
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s8))
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; GFX9: $vgpr0 = COPY [[AND]](s32)
     %0:_(p0) = COPY $vgpr0_vgpr1
     %1:_(s2) = G_LOAD %0 :: (load (s2), align 1, addrspace 0)
     %2:_(s32) = G_ZEXT %1
     $vgpr0 = COPY %2

llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir

 ---
 name: test_load_global_s1_align1
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; SI-LABEL: name: test_load_global_s1_align1
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s1), addrspace 1)
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; SI: $vgpr0 = COPY [[AND]](s32)
     ; CI-HSA-LABEL: name: test_load_global_s1_align1
     ; CI-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI-HSA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s1), addrspace 1)
+    ; CI-HSA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
     ; CI-HSA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CI-HSA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; CI-HSA: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CI-HSA: $vgpr0 = COPY [[AND]](s32)
     ; CI-MESA-LABEL: name: test_load_global_s1_align1
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s1), addrspace 1)
+    ; CI-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
     ; CI-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CI-MESA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; CI-MESA: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CI-MESA: $vgpr0 = COPY [[AND]](s32)
     ; VI-LABEL: name: test_load_global_s1_align1
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s1), addrspace 1)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; VI: $vgpr0 = COPY [[AND]](s32)
     ; GFX9-HSA-LABEL: name: test_load_global_s1_align1
     ; GFX9-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9-HSA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s1), addrspace 1)
+    ; GFX9-HSA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
     ; GFX9-HSA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; GFX9-HSA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; GFX9-HSA: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; GFX9-HSA: $vgpr0 = COPY [[AND]](s32)
     ; GFX9-MESA-LABEL: name: test_load_global_s1_align1
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s1), addrspace 1)
+    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
     ; GFX9-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; GFX9-MESA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; GFX9-MESA: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; GFX9-MESA: $vgpr0 = COPY [[AND]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s1) = G_LOAD %0 :: (load (s1), align 1, addrspace 1)
     %2:_(s32) = G_ZEXT %1
     $vgpr0 = COPY %2
 ...
 
 ---
 name: test_load_global_s2_align1
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; SI-LABEL: name: test_load_global_s2_align1
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s2), addrspace 1)
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; SI: $vgpr0 = COPY [[AND]](s32)
     ; CI-HSA-LABEL: name: test_load_global_s2_align1
     ; CI-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI-HSA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s2), addrspace 1)
+    ; CI-HSA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
     ; CI-HSA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; CI-HSA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; CI-HSA: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CI-HSA: $vgpr0 = COPY [[AND]](s32)
     ; CI-MESA-LABEL: name: test_load_global_s2_align1
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s2), addrspace 1)
+    ; CI-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
     ; CI-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; CI-MESA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; CI-MESA: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CI-MESA: $vgpr0 = COPY [[AND]](s32)
     ; VI-LABEL: name: test_load_global_s2_align1
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s2), addrspace 1)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; VI: $vgpr0 = COPY [[AND]](s32)
     ; GFX9-HSA-LABEL: name: test_load_global_s2_align1
     ; GFX9-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9-HSA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s2), addrspace 1)
+    ; GFX9-HSA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
     ; GFX9-HSA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; GFX9-HSA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; GFX9-HSA: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; GFX9-HSA: $vgpr0 = COPY [[AND]](s32)
     ; GFX9-MESA-LABEL: name: test_load_global_s2_align1
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s2), addrspace 1)
+    ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
     ; GFX9-MESA: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; GFX9-MESA: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; GFX9-MESA: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; GFX9-MESA: $vgpr0 = COPY [[AND]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s2) = G_LOAD %0 :: (load (s2), align 1, addrspace 1)
     %2:_(s32) = G_ZEXT %1
     $vgpr0 = COPY %2

llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -global-isel-abort=0  %s -o - | FileCheck -check-prefix=SI %s
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=bonaire -O0 -run-pass=legalizer -global-isel-abort=0  %s -o - | FileCheck -check-prefix=CI %s
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=VI %s
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=GFX9 %s
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=GFX9 %s
 
 ---
 name: test_load_private_s1_align1
 body: |
   bb.0:
     liveins: $vgpr0
 
     ; SI-LABEL: name: test_load_private_s1_align1
     ; SI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s1), addrspace 5)
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s8), addrspace 5)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; SI: $vgpr0 = COPY [[AND]](s32)
     ; CI-LABEL: name: test_load_private_s1_align1
     ; CI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s1), addrspace 5)
+    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s8), addrspace 5)
     ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; CI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CI: $vgpr0 = COPY [[AND]](s32)
     ; VI-LABEL: name: test_load_private_s1_align1
     ; VI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s1), addrspace 5)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s8), addrspace 5)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; VI: $vgpr0 = COPY [[AND]](s32)
     ; GFX9-LABEL: name: test_load_private_s1_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s1), addrspace 5)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s8), addrspace 5)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; GFX9: $vgpr0 = COPY [[AND]](s32)
     %0:_(p5) = COPY $vgpr0
     %1:_(s1) = G_LOAD %0 :: (load (s1), align 1, addrspace 5)
     %2:_(s32) = G_ZEXT %1
     $vgpr0 = COPY %2
 ...
 
 ---
 name: test_load_private_s2_align1
 body: |
   bb.0:
     liveins: $vgpr0
 
     ; SI-LABEL: name: test_load_private_s2_align1
     ; SI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s2), addrspace 5)
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s8), addrspace 5)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; SI: $vgpr0 = COPY [[AND]](s32)
     ; CI-LABEL: name: test_load_private_s2_align1
     ; CI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s2), addrspace 5)
+    ; CI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s8), addrspace 5)
     ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; CI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CI: $vgpr0 = COPY [[AND]](s32)
     ; VI-LABEL: name: test_load_private_s2_align1
     ; VI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s2), addrspace 5)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s8), addrspace 5)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; VI: $vgpr0 = COPY [[AND]](s32)
     ; GFX9-LABEL: name: test_load_private_s2_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s2), addrspace 5)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s8), addrspace 5)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; GFX9: $vgpr0 = COPY [[AND]](s32)
     %0:_(p5) = COPY $vgpr0
     %1:_(s2) = G_LOAD %0 :: (load (s2), align 1, addrspace 5)
     %2:_(s32) = G_ZEXT %1
     $vgpr0 = COPY %2

llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sextload-global.mir

 ---
 name: test_sextload_global_i32_i1
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; GFX8-LABEL: name: test_sextload_global_i32_i1
     ; GFX8: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX8: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p1) :: (load (s1), addrspace 1)
-    ; GFX8: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    ; GFX8: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
+    ; GFX8: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LOAD]], 1
+    ; GFX8: $vgpr0 = COPY [[SEXT_INREG]](s32)
     ; GFX6-LABEL: name: test_sextload_global_i32_i1
     ; GFX6: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX6: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p1) :: (load (s1), addrspace 1)
-    ; GFX6: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    ; GFX6: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
+    ; GFX6: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LOAD]], 1
+    ; GFX6: $vgpr0 = COPY [[SEXT_INREG]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_SEXTLOAD %0 :: (load (s1), addrspace 1)
     $vgpr0 = COPY %1
 ...
 
 ---
 name: test_sextload_global_i32_i7
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; GFX8-LABEL: name: test_sextload_global_i32_i7
     ; GFX8: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX8: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p1) :: (load (s7), addrspace 1)
-    ; GFX8: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    ; GFX8: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
+    ; GFX8: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LOAD]], 7
+    ; GFX8: $vgpr0 = COPY [[SEXT_INREG]](s32)
     ; GFX6-LABEL: name: test_sextload_global_i32_i7
     ; GFX6: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX6: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p1) :: (load (s7), addrspace 1)
-    ; GFX6: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    ; GFX6: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
+    ; GFX6: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LOAD]], 7
+    ; GFX6: $vgpr0 = COPY [[SEXT_INREG]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_SEXTLOAD %0 :: (load (s7), addrspace 1)
     $vgpr0 = COPY %1
 ...
 ---
 name: test_sextload_global_i32_i24
 body: |
   bb.0:

 ---
 name: test_sextload_global_i32_i30
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; GFX8-LABEL: name: test_sextload_global_i32_i30
     ; GFX8: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX8: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p1) :: (load (s30), addrspace 1)
-    ; GFX8: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    ; GFX8: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s32), addrspace 1)
+    ; GFX8: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LOAD]], 30
+    ; GFX8: $vgpr0 = COPY [[SEXT_INREG]](s32)
     ; GFX6-LABEL: name: test_sextload_global_i32_i30
     ; GFX6: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX6: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p1) :: (load (s30), addrspace 1)
-    ; GFX6: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    ; GFX6: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s32), addrspace 1)
+    ; GFX6: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LOAD]], 30
+    ; GFX6: $vgpr0 = COPY [[SEXT_INREG]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_SEXTLOAD %0 :: (load (s30), addrspace 1)
     $vgpr0 = COPY %1
 ...
 
 ---
 name: test_sextload_global_i32_i31
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; GFX8-LABEL: name: test_sextload_global_i32_i31
     ; GFX8: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX8: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p1) :: (load (s31), addrspace 1)
-    ; GFX8: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    ; GFX8: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s32), addrspace 1)
+    ; GFX8: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LOAD]], 31
+    ; GFX8: $vgpr0 = COPY [[SEXT_INREG]](s32)
     ; GFX6-LABEL: name: test_sextload_global_i32_i31
     ; GFX6: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX6: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p1) :: (load (s31), addrspace 1)
-    ; GFX6: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    ; GFX6: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s32), addrspace 1)
+    ; GFX6: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LOAD]], 31
+    ; GFX6: $vgpr0 = COPY [[SEXT_INREG]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_SEXTLOAD %0 :: (load (s31), addrspace 1)
     $vgpr0 = COPY %1
 ...
 
 ---
 name: test_sextload_global_i32_i8
 body: |

llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir

bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2
 
     ; SI-LABEL: name: test_store_global_s1_align1
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
     ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; SI: G_STORE [[AND]](s32), [[COPY]](p1) :: (store (s1), addrspace 1)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
+    ; SI: G_STORE [[AND1]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     ; CI-LABEL: name: test_store_global_s1_align1
     ; CI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
     ; CI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; CI: G_STORE [[AND]](s32), [[COPY]](p1) :: (store (s1), addrspace 1)
+    ; CI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
+    ; CI: G_STORE [[AND1]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     ; VI-LABEL: name: test_store_global_s1_align1
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
     ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; VI: G_STORE [[AND]](s32), [[COPY]](p1) :: (store (s1), addrspace 1)
+    ; VI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
+    ; VI: G_STORE [[AND1]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     ; GFX9-LABEL: name: test_store_global_s1_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
     ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; GFX9: G_STORE [[AND]](s32), [[COPY]](p1) :: (store (s1), addrspace 1)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
+    ; GFX9: G_STORE [[AND1]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = COPY $vgpr2
     %2:_(s1) = G_TRUNC %1
     G_STORE %2, %0 :: (store (s1), align 1, addrspace 1)
 ...
 
 ---
 name: test_store_global_s7_align1
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2
 
     ; SI-LABEL: name: test_store_global_s7_align1
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
-    ; SI: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s7), addrspace 1)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: G_STORE [[AND]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     ; CI-LABEL: name: test_store_global_s7_align1
     ; CI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; CI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
-    ; CI: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s7), addrspace 1)
+    ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; CI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; CI: G_STORE [[AND]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     ; VI-LABEL: name: test_store_global_s7_align1
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
-    ; VI: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s7), addrspace 1)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; VI: G_STORE [[AND]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     ; GFX9-LABEL: name: test_store_global_s7_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
-    ; GFX9: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s7), addrspace 1)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; GFX9: G_STORE [[AND]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = COPY $vgpr2
     %2:_(s7) = G_TRUNC %1
     G_STORE %2, %0 :: (store (s7), align 1, addrspace 1)
 ...
 
 ---
 name: test_store_global_s8_align1

 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2
 
     ; SI-LABEL: name: test_store_global_s25_align4
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
-    ; SI: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s25), addrspace 1)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 33554431
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: G_STORE [[AND]](s32), [[COPY]](p1) :: (store (s32), addrspace 1)
     ; CI-LABEL: name: test_store_global_s25_align4
     ; CI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; CI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
-    ; CI: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s25), addrspace 1)
+    ; CI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 33554431
+    ; CI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; CI: G_STORE [[AND]](s32), [[COPY]](p1) :: (store (s32), addrspace 1)
     ; VI-LABEL: name: test_store_global_s25_align4
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
-    ; VI: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s25), addrspace 1)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 33554431
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; VI: G_STORE [[AND]](s32), [[COPY]](p1) :: (store (s32), addrspace 1)
     ; GFX9-LABEL: name: test_store_global_s25_align4
     ; GFX9: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
-    ; GFX9: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s25), addrspace 1)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 33554431
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; GFX9: G_STORE [[AND]](s32), [[COPY]](p1) :: (store (s32), addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = COPY $vgpr2
     %2:_(s25) = G_TRUNC %1
     G_STORE %2, %0 :: (store (s25), align 4, addrspace 1)
 ...
 
 # ---
 # name: test_store_global_s25_align2

llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir

bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2
 
     ; SI-LABEL: name: test_store_global_i1
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
     ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; SI: G_STORE [[AND]](s32), [[COPY]](p1) :: (store (s1), addrspace 1)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
+    ; SI: G_STORE [[AND1]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     ; VI-LABEL: name: test_store_global_i1
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
     ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; VI: G_STORE [[AND]](s32), [[COPY]](p1) :: (store (s1), addrspace 1)
+    ; VI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
+    ; VI: G_STORE [[AND1]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = COPY $vgpr2
     %2:_(s1) = G_TRUNC %1
     G_STORE %2, %0 :: (store (s1), addrspace 1)
 ...
 
 ---
 name: test_store_global_i8
bb.0:
     ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
     ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
     ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
     ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
     ; SI: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
     ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
-    ; SI: G_STORE [[COPY5]](s32), [[COPY]](p1) :: (store (<3 x s2>), addrspace 1)
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; SI: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C4]]
+    ; SI: G_STORE [[AND4]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     ; VI-LABEL: name: test_truncstore_global_v3s8_to_1_align1
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
     ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
     ; VI: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
     ; VI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
     ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
bb.0:
     ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C1]](s16)
     ; VI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
     ; VI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
     ; VI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
     ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
     ; VI: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
     ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
-    ; VI: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (<3 x s2>), addrspace 1)
+    ; VI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; VI: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C3]]
+    ; VI: G_STORE [[AND4]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
     %2:_(<3 x s8>) = G_TRUNC %1
     G_STORE %2, %0 :: (store (<3 x s2>), addrspace 1, align 1)
 
 ...
 
 ---
bb.0:
     ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
     ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
     ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
     ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
     ; SI: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
     ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
-    ; SI: G_STORE [[COPY5]](s32), [[COPY]](p1) :: (store (<3 x s4>), addrspace 1)
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4095
+    ; SI: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C4]]
+    ; SI: G_STORE [[AND4]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
     ; VI-LABEL: name: test_truncstore_global_v3s8_to_2_align2
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
     ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
     ; VI: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
     ; VI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
     ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
bb.0:
     ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C1]](s16)
     ; VI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
     ; VI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
     ; VI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
     ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
     ; VI: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
     ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
-    ; VI: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (<3 x s4>), addrspace 1)
+    ; VI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4095
+    ; VI: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C3]]
+    ; VI: G_STORE [[AND4]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
     %2:_(<3 x s8>) = G_TRUNC %1
     G_STORE %2, %0 :: (store (<3 x s4>), addrspace 1, align 2)
 
 ...

llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-zextload-global.mir

 ---
 name: test_zextload_global_i32_i1
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; GFX8-LABEL: name: test_zextload_global_i32_i1
     ; GFX8: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX8: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p1) :: (load (s1), addrspace 1)
-    ; GFX8: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    ; GFX8: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
+    ; GFX8: [[ASSERT_ZEXT:%[0-9]+]]:_(s32) = G_ASSERT_ZEXT [[LOAD]], 1
+    ; GFX8: $vgpr0 = COPY [[ASSERT_ZEXT]](s32)
     ; GFX6-LABEL: name: test_zextload_global_i32_i1
     ; GFX6: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX6: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p1) :: (load (s1), addrspace 1)
-    ; GFX6: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    ; GFX6: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
+    ; GFX6: [[ASSERT_ZEXT:%[0-9]+]]:_(s32) = G_ASSERT_ZEXT [[LOAD]], 1
+    ; GFX6: $vgpr0 = COPY [[ASSERT_ZEXT]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_ZEXTLOAD %0 :: (load (s1), addrspace 1)
     $vgpr0 = COPY %1
 ...
 
 ---
 name: test_zextload_global_i32_i7
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; GFX8-LABEL: name: test_zextload_global_i32_i7
     ; GFX8: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX8: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p1) :: (load (s7), addrspace 1)
-    ; GFX8: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    ; GFX8: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
+    ; GFX8: [[ASSERT_ZEXT:%[0-9]+]]:_(s32) = G_ASSERT_ZEXT [[LOAD]], 7
+    ; GFX8: $vgpr0 = COPY [[ASSERT_ZEXT]](s32)
     ; GFX6-LABEL: name: test_zextload_global_i32_i7
     ; GFX6: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX6: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p1) :: (load (s7), addrspace 1)
-    ; GFX6: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    ; GFX6: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s8), addrspace 1)
+    ; GFX6: [[ASSERT_ZEXT:%[0-9]+]]:_(s32) = G_ASSERT_ZEXT [[LOAD]], 7
+    ; GFX6: $vgpr0 = COPY [[ASSERT_ZEXT]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_ZEXTLOAD %0 :: (load (s7), addrspace 1)
     $vgpr0 = COPY %1
 ...
 
 ---
 name: test_zextload_global_i32_i24
 body: |

 ---
 name: test_zextload_global_i32_i30
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; GFX8-LABEL: name: test_zextload_global_i32_i30
     ; GFX8: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX8: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p1) :: (load (s30), addrspace 1)
-    ; GFX8: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    ; GFX8: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s32), addrspace 1)
+    ; GFX8: [[ASSERT_ZEXT:%[0-9]+]]:_(s32) = G_ASSERT_ZEXT [[LOAD]], 30
+    ; GFX8: $vgpr0 = COPY [[ASSERT_ZEXT]](s32)
     ; GFX6-LABEL: name: test_zextload_global_i32_i30
     ; GFX6: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX6: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p1) :: (load (s30), addrspace 1)
-    ; GFX6: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    ; GFX6: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s32), addrspace 1)
+    ; GFX6: [[ASSERT_ZEXT:%[0-9]+]]:_(s32) = G_ASSERT_ZEXT [[LOAD]], 30
+    ; GFX6: $vgpr0 = COPY [[ASSERT_ZEXT]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_ZEXTLOAD %0 :: (load (s30), addrspace 1)
     $vgpr0 = COPY %1
 ...
 
 ---
 name: test_zextload_global_i32_i31
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
     ; GFX8-LABEL: name: test_zextload_global_i32_i31
     ; GFX8: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX8: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p1) :: (load (s31), addrspace 1)
-    ; GFX8: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    ; GFX8: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s32), addrspace 1)
+    ; GFX8: [[ASSERT_ZEXT:%[0-9]+]]:_(s32) = G_ASSERT_ZEXT [[LOAD]], 31
+    ; GFX8: $vgpr0 = COPY [[ASSERT_ZEXT]](s32)
     ; GFX6-LABEL: name: test_zextload_global_i32_i31
     ; GFX6: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX6: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p1) :: (load (s31), addrspace 1)
-    ; GFX6: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    ; GFX6: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load (s32), addrspace 1)
+    ; GFX6: [[ASSERT_ZEXT:%[0-9]+]]:_(s32) = G_ASSERT_ZEXT [[LOAD]], 31
+    ; GFX6: $vgpr0 = COPY [[ASSERT_ZEXT]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_ZEXTLOAD %0 :: (load (s31), addrspace 1)
     $vgpr0 = COPY %1
 ...
 
 ---
 name: test_zextload_global_i32_i8
 body: |

llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll

 ; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 1, v2
 ; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s7, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
 ; GFX8-NEXT:    v_subrev_u32_e64 v4, s[0:1], s7, v3
 ; GFX8-NEXT:    v_xor_b32_e32 v2, s4, v2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, s4, v2
 ; GFX8-NEXT:    v_xor_b32_e32 v3, s8, v3
+; GFX8-NEXT:    v_and_b32_e32 v2, 7, v2
 ; GFX8-NEXT:    flat_store_byte v[0:1], v2
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, s8, v3
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_and_b32_e32 v2, 7, v3
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    flat_store_byte v[0:1], v3
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: sdivrem_i3:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x10
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_lshr_b32 s1, s0, 8

 ; GFX9-NEXT:    v_subrev_u32_e32 v3, s7, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GFX9-NEXT:    v_add_u32_e32 v3, 1, v0
 ; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s7, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GFX9-NEXT:    v_subrev_u32_e32 v3, s7, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GFX9-NEXT:    v_xor_b32_e32 v0, s4, v0
-; GFX9-NEXT:    v_xor_b32_e32 v1, s8, v1
 ; GFX9-NEXT:    v_subrev_u32_e32 v0, s4, v0
+; GFX9-NEXT:    v_xor_b32_e32 v1, s8, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 7, v0
 ; GFX9-NEXT:    v_subrev_u32_e32 v1, s8, v1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    global_store_byte v2, v0, s[0:1]
-; GFX9-NEXT:    global_store_byte v2, v1, s[2:3]
+; GFX9-NEXT:    v_and_b32_e32 v0, 7, v1
+; GFX9-NEXT:    global_store_byte v2, v0, s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: sdivrem_i3:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dword s0, s[4:5], 0x10
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_lshr_b32 s1, s0, 8
 ; GFX10-NEXT:    s_bfe_i32 s0, s0, 0x30000
 ; GFX10-NEXT:    s_bfe_i32 s1, s1, 0x30000
-; GFX10-NEXT:    s_ashr_i32 s8, s0, 31
+; GFX10-NEXT:    s_ashr_i32 s7, s0, 31
 ; GFX10-NEXT:    s_ashr_i32 s6, s1, 31
-; GFX10-NEXT:    s_add_i32 s0, s0, s8
+; GFX10-NEXT:    s_add_i32 s0, s0, s7
 ; GFX10-NEXT:    s_add_i32 s1, s1, s6
-; GFX10-NEXT:    s_xor_b32 s0, s0, s8
-; GFX10-NEXT:    s_xor_b32 s7, s1, s6
-; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s7
-; GFX10-NEXT:    s_sub_i32 s1, 0, s7
+; GFX10-NEXT:    s_xor_b32 s0, s0, s7
+; GFX10-NEXT:    s_xor_b32 s1, s1, s6
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s1
+; GFX10-NEXT:    s_sub_i32 s2, 0, s1
 ; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT:    v_mul_lo_u32 v1, s1, v0
+; GFX10-NEXT:    v_mul_lo_u32 v1, s2, v0
 ; GFX10-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
 ; GFX10-NEXT:    v_mul_hi_u32 v0, s0, v0
-; GFX10-NEXT:    v_mul_lo_u32 v1, v0, s7
+; GFX10-NEXT:    v_mul_lo_u32 v1, v0, s1
 ; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v0
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v1, s0, v1
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX10-NEXT:    s_xor_b32 s4, s8, s6
-; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s7, v1
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, s7, v1
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s1, v1
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, s1, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v0
-; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s7, v1
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, s7, v1
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s1, v1
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, s1, v1
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-NEXT:    s_xor_b32 s4, s7, s6
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    v_xor_b32_e32 v0, s4, v0
-; GFX10-NEXT:    v_xor_b32_e32 v1, s8, v1
+; GFX10-NEXT:    v_xor_b32_e32 v1, s7, v1
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v0, s4, v0
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v1, s8, v1
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v1, s7, v1
+; GFX10-NEXT:    v_and_b32_e32 v0, 7, v0
+; GFX10-NEXT:    v_and_b32_e32 v1, 7, v1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_store_byte v2, v0, s[0:1]
 ; GFX10-NEXT:    global_store_byte v2, v1, s[2:3]
 ; GFX10-NEXT:    s_endpgm
   %div = sdiv i3 %x, %y
   store i3 %div, i3 addrspace(1)* %out0
   %rem = srem i3 %x, %y
   store i3 %rem, i3 addrspace(1)* %out1
   ret void
 }
 
-; FIXME: Reenable test
-; define amdgpu_kernel void @sdivrem_i27(i27 addrspace(1)* %out0, i27 addrspace(1)* %out1, i27 %x, i27 %y) {
-;   %div = sdiv i27 %x, %y
-;   store i27 %div, i27 addrspace(1)* %out0
-;   %rem = srem i27 %x, %y
-;   store i27 %rem, i27 addrspace(1)* %out1
-;   ret void
-; }
+define amdgpu_kernel void @sdivrem_i27(i27 addrspace(1)* %out0, i27 addrspace(1)* %out1, i27 %x, i27 %y) {
+; GFX8-LABEL: sdivrem_i27:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x10
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
+; GFX8-NEXT:    s_mov_b32 s9, 0x7ffffff
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_bfe_i32 s1, s1, 0x1b0000
+; GFX8-NEXT:    s_ashr_i32 s2, s1, 31
+; GFX8-NEXT:    s_add_i32 s1, s1, s2
+; GFX8-NEXT:    s_xor_b32 s3, s1, s2
+; GFX8-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GFX8-NEXT:    s_sub_i32 s1, 0, s3
+; GFX8-NEXT:    s_bfe_i32 s0, s0, 0x1b0000
+; GFX8-NEXT:    s_ashr_i32 s8, s0, 31
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT:    s_add_i32 s0, s0, s8
+; GFX8-NEXT:    s_xor_b32 s0, s0, s8
+; GFX8-NEXT:    s_xor_b32 s2, s8, s2
+; GFX8-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_lo_u32 v1, s1, v0
+; GFX8-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GFX8-NEXT:    v_mul_lo_u32 v1, v0, s3
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v0
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, s0, v1
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s3, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_subrev_u32_e64 v2, s[0:1], s3, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s3, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_subrev_u32_e64 v2, s[0:1], s3, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX8-NEXT:    v_xor_b32_e32 v0, s2, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT:    v_xor_b32_e32 v1, s8, v1
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, s8, v1
+; GFX8-NEXT:    v_and_b32_e32 v3, s9, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    flat_store_dword v[0:1], v3
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_and_b32_e32 v2, s9, v2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: sdivrem_i27:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x10
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_bfe_i32 s1, s1, 0x1b0000
+; GFX9-NEXT:    s_ashr_i32 s6, s1, 31
+; GFX9-NEXT:    s_add_i32 s1, s1, s6
+; GFX9-NEXT:    s_xor_b32 s7, s1, s6
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s7
+; GFX9-NEXT:    s_sub_i32 s1, 0, s7
+; GFX9-NEXT:    s_bfe_i32 s0, s0, 0x1b0000
+; GFX9-NEXT:    s_ashr_i32 s8, s0, 31
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    s_add_i32 s0, s0, s8
+; GFX9-NEXT:    s_xor_b32 s9, s0, s8
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, s1, v0
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    s_xor_b32 s5, s8, s6
+; GFX9-NEXT:    s_mov_b32 s4, 0x7ffffff
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_mul_hi_u32 v0, s9, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, v0, s7
+; GFX9-NEXT:    v_add_u32_e32 v3, 1, v0
+; GFX9-NEXT:    v_sub_u32_e32 v1, s9, v1
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s7, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v3, s7, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_add_u32_e32 v3, 1, v0
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s7, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v3, s7, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v0, s5, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v0, s5, v0
+; GFX9-NEXT:    v_xor_b32_e32 v1, s8, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v1, s8, v1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]
+; GFX9-NEXT:    v_and_b32_e32 v0, s4, v1
+; GFX9-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: sdivrem_i27:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x10
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_bfe_i32 s1, s1, 0x1b0000
+; GFX10-NEXT:    s_bfe_i32 s0, s0, 0x1b0000
+; GFX10-NEXT:    s_ashr_i32 s6, s1, 31
+; GFX10-NEXT:    s_ashr_i32 s7, s0, 31
+; GFX10-NEXT:    s_add_i32 s1, s1, s6
+; GFX10-NEXT:    s_add_i32 s0, s0, s7
+; GFX10-NEXT:    s_xor_b32 s1, s1, s6
+; GFX10-NEXT:    s_xor_b32 s0, s0, s7
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s1
+; GFX10-NEXT:    s_sub_i32 s2, 0, s1
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX10-NEXT:    v_mul_lo_u32 v1, s2, v0
+; GFX10-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GFX10-NEXT:    v_mul_lo_u32 v1, v0, s1
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v0
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, s0, v1
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s1, v1
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, s1, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v0
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s1, v1
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, s1, v1
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-NEXT:    s_xor_b32 s4, s7, s6
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX10-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-NEXT:    v_xor_b32_e32 v0, s4, v0
+; GFX10-NEXT:    v_xor_b32_e32 v1, s7, v1
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v0, s4, v0
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v1, s7, v1
+; GFX10-NEXT:    s_mov_b32 s4, 0x7ffffff
+; GFX10-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX10-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_store_dword v2, v0, s[0:1]
+; GFX10-NEXT:    global_store_dword v2, v1, s[2:3]
+; GFX10-NEXT:    s_endpgm
+  %div = sdiv i27 %x, %y
+  store i27 %div, i27 addrspace(1)* %out0
+  %rem = srem i27 %x, %y
+  store i27 %rem, i27 addrspace(1)* %out1
+  ret void
+}

llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll

 ; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s6, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
 ; GFX8-NEXT:    v_subrev_u32_e64 v4, s[0:1], s6, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 1, v2
 ; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s6, v3
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
 ; GFX8-NEXT:    v_subrev_u32_e64 v4, s[0:1], s6, v3
+; GFX8-NEXT:    v_and_b32_e32 v2, 7, v2
 ; GFX8-NEXT:    flat_store_byte v[0:1], v2
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_and_b32_e32 v2, 7, v3
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    flat_store_byte v[0:1], v3
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: udivrem_i3:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x10
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_lshr_b32 s1, s0, 8

 ; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s6, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GFX9-NEXT:    v_subrev_u32_e32 v3, s6, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GFX9-NEXT:    v_add_u32_e32 v3, 1, v0
 ; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s6, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GFX9-NEXT:    v_subrev_u32_e32 v3, s6, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 7, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    global_store_byte v2, v0, s[0:1]
-; GFX9-NEXT:    global_store_byte v2, v1, s[2:3]
+; GFX9-NEXT:    v_and_b32_e32 v0, 7, v1
+; GFX9-NEXT:    global_store_byte v2, v0, s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: udivrem_i3:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dword s0, s[4:5], 0x10
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_lshr_b32 s1, s0, 8
 ; GFX10-NEXT:    s_and_b32 s0, s0, 7

 ; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s6, v1
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, s6, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v0
 ; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s6, v1
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, s6, v1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX10-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-NEXT:    v_and_b32_e32 v0, 7, v0
+; GFX10-NEXT:    v_and_b32_e32 v1, 7, v1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_store_byte v2, v0, s[0:1]
 ; GFX10-NEXT:    global_store_byte v2, v1, s[2:3]
 ; GFX10-NEXT:    s_endpgm
   %div = udiv i3 %x, %y
   store i3 %div, i3 addrspace(1)* %out0
   %rem = urem i3 %x, %y
   store i3 %rem, i3 addrspace(1)* %out1
   ret void
 }
 
-; FIXME: Reenable test
-; define amdgpu_kernel void @udivrem_i27(i27 addrspace(1)* %out0, i27 addrspace(1)* %out1, i27 %x, i27 %y) {
-;   %div = udiv i27 %x, %y
-;   store i27 %div, i27 addrspace(1)* %out0
-;   %rem = urem i27 %x, %y
-;   store i27 %rem, i27 addrspace(1)* %out1
-;   ret void
-; }
+define amdgpu_kernel void @udivrem_i27(i27 addrspace(1)* %out0, i27 addrspace(1)* %out1, i27 %x, i27 %y) {
+; GFX8-LABEL: udivrem_i27:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x10
+; GFX8-NEXT:    s_mov_b32 s6, 0x7ffffff
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_and_b32 s7, s1, s6
+; GFX8-NEXT:    v_cvt_f32_u32_e32 v0, s7
+; GFX8-NEXT:    s_sub_i32 s1, 0, s7
+; GFX8-NEXT:    s_and_b32 s8, s0, s6
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_lo_u32 v1, s1, v0
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX8-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_mul_hi_u32 v2, s8, v0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_mul_lo_u32 v3, v2, s7
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 1, v2
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, s8, v3
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s7, v3
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT:    v_subrev_u32_e64 v4, s[0:1], s7, v3
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 1, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, s7, v3
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT:    v_subrev_u32_e64 v4, s[0:1], s7, v3
+; GFX8-NEXT:    v_and_b32_e32 v2, s6, v2
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_and_b32_e32 v2, s6, v3
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: udivrem_i27:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x10
+; GFX9-NEXT:    s_mov_b32 s6, 0x7ffffff
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s7, s1, s6
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s7
+; GFX9-NEXT:    s_sub_i32 s1, 0, s7
+; GFX9-NEXT:    s_and_b32 s8, s0, s6
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, s1, v0
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_mul_hi_u32 v0, s8, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, v0, s7
+; GFX9-NEXT:    v_add_u32_e32 v3, 1, v0
+; GFX9-NEXT:    v_sub_u32_e32 v1, s8, v1
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s7, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v3, s7, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_add_u32_e32 v3, 1, v0
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s7, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v3, s7, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, s6, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]
+; GFX9-NEXT:    v_and_b32_e32 v0, s6, v1
+; GFX9-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: udivrem_i27:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x10
+; GFX10-NEXT:    s_mov_b32 s6, 0x7ffffff
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_and_b32 s7, s1, s6
+; GFX10-NEXT:    s_and_b32 s0, s0, s6
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s7
+; GFX10-NEXT:    s_sub_i32 s1, 0, s7
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX10-NEXT:    v_mul_lo_u32 v1, s1, v0
+; GFX10-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GFX10-NEXT:    v_mul_lo_u32 v1, v0, s7
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v0
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, s0, v1
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s7, v1
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, s7, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v0
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s7, v1
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, s7, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX10-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-NEXT:    v_and_b32_e32 v0, s6, v0
+; GFX10-NEXT:    v_and_b32_e32 v1, s6, v1
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_store_dword v2, v0, s[0:1]
+; GFX10-NEXT:    global_store_dword v2, v1, s[2:3]
+; GFX10-NEXT:    s_endpgm
+  %div = udiv i27 %x, %y
+  store i27 %div, i27 addrspace(1)* %out0
+  %rem = urem i27 %x, %y
+  store i27 %rem, i27 addrspace(1)* %out1
+  ret void
+}

llvm/test/CodeGen/ARM/GlobalISel/arm-instruction-select.mir

body:             |
   bb.0:
     liveins: $r0, $r1, $r2, $r3
 
     ; CHECK-LABEL: name: test_load_from_stack
     ; CHECK: [[ADDri:%[0-9]+]]:gpr = ADDri %fixed-stack.0, 0, 14 /* CC::al */, $noreg, $noreg
     ; CHECK: [[LDRi12_:%[0-9]+]]:gpr = LDRi12 [[ADDri]], 0, 14 /* CC::al */, $noreg :: (load (s32))
     ; CHECK: $r0 = COPY [[LDRi12_]]
     ; CHECK: [[ADDri1:%[0-9]+]]:gpr = ADDri %fixed-stack.2, 0, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: [[LDRBi12_:%[0-9]+]]:gprnopc = LDRBi12 [[ADDri1]], 0, 14 /* CC::al */, $noreg :: (load (s1))
+    ; CHECK: [[LDRBi12_:%[0-9]+]]:gprnopc = LDRBi12 [[ADDri1]], 0, 14 /* CC::al */, $noreg :: (load (s8))
     ; CHECK: [[COPY:%[0-9]+]]:gpr = COPY [[LDRBi12_]]
     ; CHECK: $r0 = COPY [[COPY]]
     ; CHECK: BX_RET 14 /* CC::al */, $noreg
     %0(p0) = G_FRAME_INDEX %fixed-stack.2
 
     %1(s32) = G_LOAD %0(p0) :: (load (s32))
 
     $r0 = COPY %1
 
     %2(p0) = G_FRAME_INDEX %fixed-stack.0
 
-    %3(s1) = G_LOAD %2(p0) :: (load (s1))
+    %3(s8) = G_LOAD %2(p0) :: (load (s8))
 
-    %4(s32) = G_ANYEXT %3(s1)
+    %4(s32) = G_ANYEXT %3(s8)
 
     $r0 = COPY %4
 
     BX_RET 14, $noreg
 ...
 ---
 name:            test_stores
 legalized:       true
 regBankSelected: true
 selected:        false
 registers:
   - { id: 0, class: gprb }
   - { id: 1, class: gprb }
   - { id: 2, class: gprb }
   - { id: 3, class: gprb }
   - { id: 4, class: gprb }
 body:             |
   bb.0:
     liveins: $r0, $r1
 
     ; CHECK-LABEL: name: test_stores
     ; CHECK: [[COPY:%[0-9]+]]:gpr = COPY $r0
     ; CHECK: [[COPY1:%[0-9]+]]:gpr = COPY $r1
     ; CHECK: [[COPY2:%[0-9]+]]:gprnopc = COPY [[COPY1]]
-    ; CHECK: [[ANDri:%[0-9]+]]:gprnopc = ANDri [[COPY1]], 1, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: STRBi12 [[ANDri]], [[COPY]], 0, 14 /* CC::al */, $noreg :: (store (s1))
     ; CHECK: STRBi12 [[COPY2]], [[COPY]], 0, 14 /* CC::al */, $noreg :: (store (s8))
     ; CHECK: STRH [[COPY1]], [[COPY]], $noreg, 0, 14 /* CC::al */, $noreg :: (store (s16))
     ; CHECK: STRi12 [[COPY1]], [[COPY]], 0, 14 /* CC::al */, $noreg :: (store (s32))
     ; CHECK: BX_RET 14 /* CC::al */, $noreg
     %0(p0) = COPY $r0
 
     %3(s32) = COPY $r1
 
-    %4(s1) = G_TRUNC %3(s32)
-
     %1(s8) = G_TRUNC %3(s32)
 
     %2(s16) = G_TRUNC %3(s32)
 
-    G_STORE %4(s1), %0(p0) :: (store (s1))
-
     G_STORE %1(s8), %0(p0) :: (store (s8))
 
     G_STORE %2(s16), %0(p0) :: (store (s16))
 
     G_STORE %3(s32), %0(p0) :: (store (s32))
 
     BX_RET 14, $noreg
 ...

llvm/test/CodeGen/ARM/GlobalISel/arm-legalize-consts.mir

bb.0:
 
     %2(s8) = G_CONSTANT i8 10
     G_STORE %2(s8), %4(p0) :: (store (s8))
     ; CHECK-NOT: G_CONSTANT i8
     ; CHECK: [[EXT:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
     ; CHECK: {{%[0-9]+}}:_(s8) = G_TRUNC [[EXT]](s32)
     ; CHECK-NOT: G_CONSTANT i8
 
-    %3(s1) = G_CONSTANT i1 1
-    G_STORE %3(s1), %4(p0) :: (store (s1))
+    %3:_(s1) = G_CONSTANT i1 1
+    %6:_(s32) = G_CONSTANT i32 99
+    %7:_(s32) = G_SELECT %3, %0, %6
+    G_STORE %7(s32), %4(p0) :: (store (s32))
     ; CHECK-NOT: G_CONSTANT i1
     ; CHECK: [[EXT:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CHECK: {{%[0-9]+}}:_(s1) = G_TRUNC [[EXT]](s32)
     ; CHECK-NOT: G_CONSTANT i1
 
     %5(p0) = G_CONSTANT i32 0
+
     G_STORE %5(p0), %4(p0) :: (store (p0))
     ; CHECK: {{%[0-9]+}}:_(p0) = G_CONSTANT i32 0
 
     $r0 = COPY %0(s32)
     BX_RET 14, $noreg, implicit $r0
 ...

llvm/test/CodeGen/ARM/GlobalISel/arm-legalize-exts.mir

 ---
 name:            test_sext_s1_to_s16
 # CHECK-LABEL: name: test_sext_s1_to_s16
 legalized:       false
 # CHECK: legalized: true
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
-registers:
-  - { id: 0, class: _ }
-  - { id: 1, class: _ }
-  - { id: 2, class: _ }
 body:             |
   bb.0:
-    liveins: $r0
+    liveins: $r0, $r1
 
-    %0(p0) = COPY $r0
-    %1(s1) = G_LOAD %0(p0) :: (load (s1))
-    %2(s16) = G_SEXT %1(s1)
+    %0:_(p0) = COPY $r0
+    %1:_(s32) = COPY $r1
+    %2:_(s32) = G_CONSTANT i32 0
+    %3:_(s1) = G_ICMP intpred(eq), %1, %2
+    %4:_(s16) = G_SEXT %3(s1)
     ; G_SEXT from s1 to s16 is legal, so we should find it unchanged in the output
     ; CHECK: {{%[0-9]+}}:_(s16) = G_SEXT {{%[0-9]+}}(s1)
-    G_STORE %2(s16), %0(p0) :: (store (s16))
+    G_STORE %4(s16), %0(p0) :: (store (s16))
     BX_RET 14, $noreg
 ...
 ---
 name:            test_anyext_s1_to_s8
 # CHECK-LABEL: name: test_anyext_s1_to_s8
 legalized:       false
 # CHECK: legalized: true
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
-registers:
-  - { id: 0, class: _ }
-  - { id: 1, class: _ }
-  - { id: 2, class: _ }
 body:             |
   bb.0:
-    liveins: $r0
+    liveins: $r0, $r1
 
-    %0(p0) = COPY $r0
-    %1(s1) = G_LOAD %0(p0) :: (load (s1))
-    %2(s8) = G_ANYEXT %1
+    %0:_(p0) = COPY $r0
+    %1:_(s32) = COPY $r1
+    %2:_(s32) = G_CONSTANT i32 0
+    %3:_(s1) = G_ICMP intpred(eq), %1, %2
+    %4:_(s8) = G_ANYEXT %3
     ; G_ANYEXT from s1 to s8 is legal, so we should find it unchanged in the output
     ; CHECK: {{%[0-9]+}}:_(s8) = G_ANYEXT {{%[0-9]+}}(s1)
-    G_STORE %2(s8), %0(p0) :: (store (s8))
+    G_STORE %4(s8), %0(p0) :: (store (s8))
     BX_RET 14, $noreg
 ...
 ---
 name:            test_ext_combine
 # CHECK-LABEL: name: test_ext_combine
 legalized:       false
 # CHECK: legalized: true
 regBankSelected: false

llvm/test/CodeGen/ARM/GlobalISel/arm-legalize-load-store.mir

 # RUN: llc -mtriple arm-- -run-pass=legalizer %s -o - | FileCheck %s
 # RUN: llc -mtriple thumbv7-- -run-pass=legalizer %s -o - | FileCheck %s
 --- |
   define void @test_legal_loads_stores() { ret void }
+  define void @test_load_store_s1() { ret void }
   define void @test_load_from_stack() { ret void }
 
   define void @test_load_store_64_vfp() #0 { ret void }
   define void @test_load_store_64_novfp() #1 { ret void }
 
   define void @test_gep_s32() { ret void }
   define void @test_gep_s16() { ret void }
 
registers:
   - { id: 4, class: _ }
   - { id: 5, class: _ }
   - { id: 6, class: _ }
 body:             |
   bb.0:
     liveins: $r0
 
     ; These are all legal, so we should find them unchanged in the output
-    ; CHECK-DAG: G_STORE {{%[0-9]+}}(s32), %0(p0)
-    ; CHECK-DAG: G_STORE {{%[0-9]+}}(s16), %0(p0)
-    ; CHECK-DAG: G_STORE {{%[0-9]+}}(s8), %0(p0)
-    ; CHECK-DAG: G_STORE {{%[0-9]+}}(s1), %0(p0)
+    ; CHECK-DAG: G_STORE {{%[0-9]+}}(s32), %0(p0) :: (store (s32))
+    ; CHECK-DAG: G_STORE {{%[0-9]+}}(s16), %0(p0) :: (store (s16))
+    ; CHECK-DAG: G_STORE {{%[0-9]+}}(s8), %0(p0) :: (store (s8))
     ; CHECK-DAG: G_STORE {{%[0-9]+}}(p0), %0(p0)
-    ; CHECK-DAG: {{%[0-9]+}}:_(s32) = G_LOAD %0(p0)
-    ; CHECK-DAG: {{%[0-9]+}}:_(s16) = G_LOAD %0(p0)
-    ; CHECK-DAG: {{%[0-9]+}}:_(s8) = G_LOAD %0(p0)
-    ; CHECK-DAG: {{%[0-9]+}}:_(s1) = G_LOAD %0(p0)
-    ; CHECK-DAG: {{%[0-9]+}}:_(p0) = G_LOAD %0(p0)
+    ; CHECK-DAG: {{%[0-9]+}}:_(s32) = G_LOAD %0(p0) :: (load (s32))
+    ; CHECK-DAG: {{%[0-9]+}}:_(s16) = G_LOAD %0(p0) :: (load (s16))
+    ; CHECK-DAG: {{%[0-9]+}}:_(s8) = G_LOAD %0(p0) :: (load (s8))
+    ; CHECK-DAG: {{%[0-9]+}}:_(p0) = G_LOAD %0(p0) :: (load (p0))
     %0(p0) = COPY $r0
     %2(s32) = G_LOAD %0(p0) :: (load (s32))
     G_STORE %2(s32), %0(p0) :: (store (s32))
     %3(s16) = G_LOAD %0(p0) :: (load (s16))
     G_STORE %3(s16), %0(p0) :: (store (s16))
     %4(s8) = G_LOAD %0(p0) :: (load (s8))
     G_STORE %4(s8), %0(p0) :: (store (s8))
-    %5(s1) = G_LOAD %0(p0) :: (load (s1))
-    G_STORE %5(s1), %0(p0) :: (store (s1))
     %6(p0) = G_LOAD %0(p0) :: (load (p0))
     G_STORE %6(p0), %0(p0) :: (store (p0))
     BX_RET 14, $noreg
 ...
+
+---
+name:            test_load_store_s1
+legalized:       false
+# CHECK: legalized: true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $r0
+
+    ; CHECK: [[LD:%[0-9]+]]:_(s8) = G_LOAD %0(p0) :: (load (s8))
+    ; CHECK: [[ONE:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[EXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LD]]
+    ; CHECK: [[COPYONE:%[0-9]+]]:_(s32) = COPY [[ONE]]
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[EXT]], [[COPYONE]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[AND]]
+    ; CHECK: G_STORE [[TRUNC]](s8), {{%[0-9]+}}(p0) :: (store (s8))
+    %0:_(p0) = COPY $r0
+    %5:_(s1) = G_LOAD %0(p0) :: (load (s1))
+    G_STORE %5(s1), %0(p0) :: (store (s1))
+    BX_RET 14, $noreg
+...
 ---
 name:            test_load_from_stack
 # CHECK-LABEL: name: test_load_from_stack
 legalized:       false
 # CHECK: legalized: true
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true

llvm/test/CodeGen/ARM/GlobalISel/arm-regbankselect.mir

 regBankSelected: false
 selected:        false
 # CHECK: registers:
 # CHECK: - { id: 0, class: gprb, preferred-register: '' }
 # CHECK: - { id: 1, class: gprb, preferred-register: '' }
 # CHECK: - { id: 2, class: gprb, preferred-register: '' }
 # CHECK: - { id: 3, class: gprb, preferred-register: '' }
 # CHECK: - { id: 4, class: gprb, preferred-register: '' }
-# CHECK: - { id: 5, class: gprb, preferred-register: '' }
-# CHECK: - { id: 6, class: fprb, preferred-register: '' }
+# CHECK: - { id: 5, class: fprb, preferred-register: '' }
 
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
   - { id: 2, class: _ }
   - { id: 3, class: _ }
   - { id: 4, class: _ }
   - { id: 5, class: _ }
   - { id: 6, class: _ }
 body:             |
   bb.0:
     liveins: $r0
     %0(p0) = COPY $r0
-    %6(s64) = G_LOAD %0 :: (load (s64))
+    %5(s64) = G_LOAD %0 :: (load (s64))
     %1(s32) = G_LOAD %0 :: (load (s32))
     %2(s16) = G_LOAD %0 :: (load (s16))
     %3(s8)  = G_LOAD %0 :: (load (s8))
-    %4(s1)  = G_LOAD %0 :: (load (s1))
-    %5(p0)  = G_LOAD %0 :: (load (p0))
+    %4(p0)  = G_LOAD %0 :: (load (p0))
     BX_RET 14, $noreg, implicit $r0
 
 ...
 ---
 name:            test_stores
 # CHECK-LABEL: name: test_stores
 legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
 # CHECK: - { id: 0, class: gprb, preferred-register: '' }
 # CHECK: - { id: 1, class: gprb, preferred-register: '' }
 # CHECK: - { id: 2, class: gprb, preferred-register: '' }
 # CHECK: - { id: 3, class: gprb, preferred-register: '' }
-# CHECK: - { id: 4, class: gprb, preferred-register: '' }
-# CHECK: - { id: 5, class: gprb, preferred-register: '' }
-# CHECK: - { id: 6, class: fprb, preferred-register: '' }
-
+# CHECK: - { id: 4, class: fprb, preferred-register: '' }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
   - { id: 2, class: _ }
   - { id: 3, class: _ }
   - { id: 4, class: _ }
   - { id: 5, class: _ }
   - { id: 6, class: _ }
 body:             |
   bb.0:
     liveins: $r0, $r1, $r5, $d6
     %0(p0) = COPY $r0
     %1(s32) = COPY $r1
     G_STORE %1(s32), %0 :: (store (s32))
     %2(s16) = G_TRUNC %1(s32)
     G_STORE %2(s16), %0 :: (store (s16))
     %3(s8) = G_TRUNC %1(s32)
     G_STORE %3(s8), %0 :: (store (s8))
-    %4(s1) = G_TRUNC %1(s32)
-    G_STORE %4(s1), %0 :: (store (s1))
-    %5(p0) = COPY $r5
-    G_STORE %5(p0), %0 :: (store (p0))
-    %6(s64) = COPY $d6
-    G_STORE %6(s64), %0 :: (store (s64))
+    %4(s64) = COPY $d6
+    G_STORE %4(s64), %0 :: (store (s64))
     BX_RET 14, $noreg, implicit $r0
 
 ...
 ---
 name:            test_stack
 # CHECK-LABEL: name: test_stack
 legalized:       true
 regBankSelected: false

llvm/test/CodeGen/ARM/GlobalISel/thumb-select-load-store.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -O0 -mtriple thumb-- -mattr=+v6t2 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
 --- |
-  define void @test_s1() { ret void }
   define void @test_s8() { ret void }
   define void @test_s16() { ret void }
   define void @test_s32() { ret void }
 
   define void @test_gep() { ret void }
 
   define void @test_load_from_stack() { ret void }
 ...
 ---
-name:            test_s1
-legalized:       true
-regBankSelected: true
-selected:        false
-registers:
-  - { id: 0, class: gprb }
-  - { id: 1, class: gprb }
-body:             |
-  bb.0:
-    liveins: $r0
-
-    ; CHECK-LABEL: name: test_s1
-    ; CHECK: [[COPY:%[0-9]+]]:gpr = COPY $r0
-    ; CHECK: [[t2LDRBi12_:%[0-9]+]]:rgpr = t2LDRBi12 [[COPY]], 0, 14 /* CC::al */, $noreg :: (load (s1))
-    ; CHECK: [[t2ANDri:%[0-9]+]]:rgpr = t2ANDri [[t2LDRBi12_]], 1, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: t2STRBi12 [[t2ANDri]], [[COPY]], 0, 14 /* CC::al */, $noreg :: (store (s1))
-    ; CHECK: BX_RET 14 /* CC::al */, $noreg
-    %0(p0) = COPY $r0
-
-    %1(s1) = G_LOAD %0(p0) :: (load (s1))
-
-    G_STORE %1(s1), %0(p0) :: (store (s1))
-
-    BX_RET 14, $noreg
-...
----
 name:            test_s8
 legalized:       true
 regBankSelected: true
 selected:        false
 registers:
   - { id: 0, class: gprb }
   - { id: 1, class: gprb }
 body:             |
body:             |
   bb.0:
     liveins: $r0, $r1, $r2, $r3
 
     ; CHECK-LABEL: name: test_load_from_stack
     ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = t2ADDri %fixed-stack.0, 0, 14 /* CC::al */, $noreg, $noreg
     ; CHECK: [[t2LDRi12_:%[0-9]+]]:gpr = t2LDRi12 [[t2ADDri]], 0, 14 /* CC::al */, $noreg :: (load (s32))
     ; CHECK: $r0 = COPY [[t2LDRi12_]]
     ; CHECK: [[t2ADDri1:%[0-9]+]]:rgpr = t2ADDri %fixed-stack.2, 0, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: [[t2LDRBi12_:%[0-9]+]]:gprnopc = t2LDRBi12 [[t2ADDri1]], 0, 14 /* CC::al */, $noreg :: (load (s1))
+    ; CHECK: [[t2LDRBi12_:%[0-9]+]]:gprnopc = t2LDRBi12 [[t2ADDri1]], 0, 14 /* CC::al */, $noreg :: (load (s8))
     ; CHECK: [[COPY:%[0-9]+]]:gpr = COPY [[t2LDRBi12_]]
     ; CHECK: $r0 = COPY [[COPY]]
     ; CHECK: BX_RET 14 /* CC::al */, $noreg
     %0(p0) = G_FRAME_INDEX %fixed-stack.2
 
     %1(s32) = G_LOAD %0(p0) :: (load (s32))
 
     $r0 = COPY %1
 
     %2(p0) = G_FRAME_INDEX %fixed-stack.0
 
-    %3(s1) = G_LOAD %2(p0) :: (load (s1))
+    %3(s8) = G_LOAD %2(p0) :: (load (s8))
 
-    %4(s32) = G_ANYEXT %3(s1)
+    %4(s32) = G_ANYEXT %3(s8)
 
     $r0 = COPY %4
 
     BX_RET 14, $noreg
 ...

llvm/test/CodeGen/Mips/GlobalISel/legalizer/add.mir

bb.1 (%ir-block.0):
     ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
     ; MIPS32: [[COPY2:%[0-9]+]]:_(p0) = COPY $a2
     ; MIPS32: [[COPY3:%[0-9]+]]:_(p0) = COPY $a3
     ; MIPS32: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY1]]
     ; MIPS32: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[COPY1]]
     ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ICMP]](s32)
     ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
-    ; MIPS32: G_STORE [[AND]](s32), [[COPY3]](p0) :: (store (s1) into %ir.pcarry_flag)
+    ; MIPS32: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
+    ; MIPS32: G_STORE [[AND1]](s32), [[COPY3]](p0) :: (store (s8) into %ir.pcarry_flag)
     ; MIPS32: G_STORE [[ADD]](s32), [[COPY2]](p0) :: (store (s32) into %ir.padd)
     ; MIPS32: RetRA
     %0:_(s32) = COPY $a0
     %1:_(s32) = COPY $a1
     %2:_(p0) = COPY $a2
     %3:_(p0) = COPY $a3
     %4:_(s32), %5:_(s1) = G_UADDO %0, %1
     G_STORE %5(s1), %3(p0) :: (store (s1) into %ir.pcarry_flag)
     G_STORE %4(s32), %2(p0) :: (store (s32) into %ir.padd)
     RetRA
 
 ...

llvm/test/CodeGen/Mips/GlobalISel/legalizer/mul.mir

bb.1 (%ir-block.0):
     ; MIPS32: [[COPY3:%[0-9]+]]:_(p0) = COPY $a3
     ; MIPS32: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[COPY1]]
     ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; MIPS32: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY]], [[COPY1]]
     ; MIPS32: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[UMULH]](s32), [[C]]
     ; MIPS32: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ICMP]](s32)
     ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C1]]
-    ; MIPS32: G_STORE [[AND]](s32), [[COPY3]](p0) :: (store (s1) into %ir.pcarry_flag)
+    ; MIPS32: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C1]]
+    ; MIPS32: G_STORE [[AND1]](s32), [[COPY3]](p0) :: (store (s8) into %ir.pcarry_flag)
     ; MIPS32: G_STORE [[MUL]](s32), [[COPY2]](p0) :: (store (s32) into %ir.pmul)
     ; MIPS32: RetRA
     %0:_(s32) = COPY $a0
     %1:_(s32) = COPY $a1
     %2:_(p0) = COPY $a2
     %3:_(p0) = COPY $a3
     %4:_(s32), %5:_(s1) = G_UMULO %0, %1
     G_STORE %5(s1), %3(p0) :: (store (s1) into %ir.pcarry_flag)
     G_STORE %4(s32), %2(p0) :: (store (s32) into %ir.pmul)
     RetRA
 
 ...

llvm/test/CodeGen/Mips/GlobalISel/legalizer/truncStore_and_aExtLoad.mir

 body:             |
   bb.1.entry:
     liveins: $a0, $a1
 
     ; MIPS32-LABEL: name: load_store_i1
     ; MIPS32: liveins: $a0, $a1
     ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
     ; MIPS32: [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
-    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY1]](p0) :: (load (s1) from %ir.py)
+    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY1]](p0) :: (load (s8) from %ir.py)
     ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; MIPS32: G_STORE [[AND]](s32), [[COPY]](p0) :: (store (s1) into %ir.px)
+    ; MIPS32: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
+    ; MIPS32: G_STORE [[AND1]](s32), [[COPY]](p0) :: (store (s8) into %ir.px)
     ; MIPS32: RetRA
     %0:_(p0) = COPY $a0
     %1:_(p0) = COPY $a1
     %2:_(s1) = G_LOAD %1(p0) :: (load (s1) from %ir.py)
     G_STORE %2(s1), %0(p0) :: (store (s1) into %ir.px)
     RetRA
 
 ...

llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/add.ll

 
 declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32)
 define void @uadd_with_overflow(i32 %lhs, i32 %rhs, i32* %padd, i1* %pcarry_flag) {
 ; MIPS32-LABEL: uadd_with_overflow:
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    addu $1, $4, $5
 ; MIPS32-NEXT:    sltu $2, $1, $5
 ; MIPS32-NEXT:    andi $2, $2, 1
+; MIPS32-NEXT:    andi $2, $2, 1
 ; MIPS32-NEXT:    sb $2, 0($7)
 ; MIPS32-NEXT:    sw $1, 0($6)
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
   %res = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %lhs, i32 %rhs)
   %carry_flag = extractvalue { i32, i1 } %res, 1
   %add = extractvalue { i32, i1 } %res, 0
   store i1 %carry_flag, i1* %pcarry_flag
   store i32 %add, i32* %padd
   ret void
 }

llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/mul.ll

 define void @umul_with_overflow(i32 %lhs, i32 %rhs, i32* %pmul, i1* %pcarry_flag) {
 ; MIPS32-LABEL: umul_with_overflow:
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    multu $4, $5
 ; MIPS32-NEXT:    mfhi $2
 ; MIPS32-NEXT:    mul $1, $4, $5
 ; MIPS32-NEXT:    sltu $2, $zero, $2
 ; MIPS32-NEXT:    andi $2, $2, 1
+; MIPS32-NEXT:    andi $2, $2, 1
 ; MIPS32-NEXT:    sb $2, 0($7)
 ; MIPS32-NEXT:    sw $1, 0($6)
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
   %res = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %lhs, i32 %rhs)
   %carry_flag = extractvalue { i32, i1 } %res, 1
   %mul = extractvalue { i32, i1 } %res, 0
   store i1 %carry_flag, i1* %pcarry_flag
   store i32 %mul, i32* %pmul
   ret void
 }

llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/truncStore_and_aExtLoad.ll

entry:
   ret i16 %0
 }
 
 define void @load_store_i1(i1* %px, i1* %py) {
 ; MIPS32-LABEL: load_store_i1:
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lbu $1, 0($5)
 ; MIPS32-NEXT:    andi $1, $1, 1
+; MIPS32-NEXT:    andi $1, $1, 1
 ; MIPS32-NEXT:    sb $1, 0($4)
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 entry:
   %0 = load i1, i1* %py
   store i1 %0, i1* %px
   ret void
 }

llvm/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp

TEST(LegalizerInfoTest, MMOAlignment) {
 
   const LLT s32 = LLT::scalar(32);
   const LLT p0 = LLT::pointer(0, 64);
 
   {
     LegalizerInfo LI;
     auto &LegacyInfo = LI.getLegacyLegalizerInfo();
     LI.getActionDefinitionsBuilder(G_LOAD)
-      .legalForTypesWithMemDesc({{s32, p0, 32, 32}});
+      .legalForTypesWithMemDesc({{s32, p0, s32, 32}});
 
     LegacyInfo.computeTables();
 
     EXPECT_ACTION(Legal, 0, LLT(),
                   LegalityQuery(G_LOAD, {s32, p0},
                                 LegalityQuery::MemDesc{
-                                  32, 32, AtomicOrdering::NotAtomic}));
+                                  s32, 32, AtomicOrdering::NotAtomic}));
     EXPECT_ACTION(Unsupported, 0, LLT(),
                   LegalityQuery(G_LOAD, {s32, p0},
                                 LegalityQuery::MemDesc{
-                                  32, 16, AtomicOrdering::NotAtomic }));
+                                  s32, 16, AtomicOrdering::NotAtomic }));
     EXPECT_ACTION(Unsupported, 0, LLT(),
                   LegalityQuery(G_LOAD, {s32, p0},
                                 LegalityQuery::MemDesc{
-                                  32, 8, AtomicOrdering::NotAtomic}));
+                                  s32, 8, AtomicOrdering::NotAtomic}));
   }
 
   // Test that the maximum supported alignment value isn't truncated
   {
     // Maximum IR defined alignment in bytes.
     const uint64_t MaxAlignment = UINT64_C(1) << 29;
     const uint64_t MaxAlignInBits = 8 * MaxAlignment;
     LegalizerInfo LI;
     auto &LegacyInfo = LI.getLegacyLegalizerInfo();
     LI.getActionDefinitionsBuilder(G_LOAD)
-      .legalForTypesWithMemDesc({{s32, p0, 32, MaxAlignInBits}});
+      .legalForTypesWithMemDesc({{s32, p0, s32, MaxAlignInBits}});
 
     LegacyInfo.computeTables();
 
     EXPECT_ACTION(Legal, 0, LLT(),
                   LegalityQuery(G_LOAD, {s32, p0},
-                                LegalityQuery::MemDesc{32,
+                                LegalityQuery::MemDesc{s32,
                                     MaxAlignInBits, AtomicOrdering::NotAtomic}));
     EXPECT_ACTION(Unsupported, 0, LLT(),
                   LegalityQuery(G_LOAD, {s32, p0},
                                 LegalityQuery::MemDesc{
-                                  32, 8, AtomicOrdering::NotAtomic }));
+                                  s32, 8, AtomicOrdering::NotAtomic }));
   }
 }
 
 // This code sequence doesn't do anything, but it covers a previously uncovered
 // codepath that used to crash in MSVC x86_32 debug mode.
 TEST(LegalizerInfoTest, MSVCDebugMiscompile) {
   const LLT S1 = LLT::scalar(1);
   const LLT P0 = LLT::pointer(0, 32);
   LegalizerInfo LI;
   auto Builder = LI.getActionDefinitionsBuilder(TargetOpcode::G_PTRTOINT);
   (void)Builder.legalForCartesianProduct({S1}, {P0});
 }

llvm/unittests/CodeGen/GlobalISel/LegalizerTest.cpp

::testing::AssertionResult isNullMIPtr(const MachineInstr *MI) {
   MI->print(MISStream, /*IsStandalone=*/true, /*SkipOpers=*/false,
             /*SkipDebugLoc=*/false, /*AddNewLine=*/false);
   return ::testing::AssertionFailure()
          << "unable to legalize instruction: " << MISStream.str();
 }
 
 DefineLegalizerInfo(ALegalizer, {
   auto p0 = LLT::pointer(0, 64);
+  auto s8 = LLT::scalar(8);
   auto v2s8 = LLT::vector(2, 8);
   auto v2s16 = LLT::vector(2, 16);
   getActionDefinitionsBuilder(G_LOAD)
-      .legalForTypesWithMemDesc({{s16, p0, 8, 8}})
+      .legalForTypesWithMemDesc({{s16, p0, s8, 8}})
       .scalarize(0)
       .clampScalar(0, s16, s16);
   getActionDefinitionsBuilder(G_PTR_ADD).legalFor({{p0, s64}});
   getActionDefinitionsBuilder(G_CONSTANT).legalFor({s32, s64});
   getActionDefinitionsBuilder(G_BUILD_VECTOR)
       .legalFor({{v2s16, s16}})
       .clampScalar(1, s16, s16);
   getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC).legalFor({{v2s8, s16}});