[AArch64][GlobalISel] Support truncstorei8/i16 w/ combine to form truncating G_STOREs.

[aemerson]

Summary

This needs some tablegen changes so that we can actually import the patterns properly.

[arsenm]

llvm/utils/TableGen/GlobalISelEmitter.cpp

3783–3785

Where does this actually come up? Are there really patterns using i1 memory types?

[aemerson]

llvm/utils/TableGen/GlobalISelEmitter.cpp

3783–3785

I’m not sure, this was just a refactoring of existing code.

[paquette]

llvm/utils/TableGen/GlobalISelEmitter.cpp

3660

should probably be a doxygen comment

3784

This comment is kind of confusing?

[paquette]

LGTM

llvm/include/llvm/Target/TargetSelectionDAG.td

def truncstore : PatFrag<(ops node:$val, node:$ptr),
                          (unindexedstore node:$val, node:$ptr)> {
   let IsStore = true;
   let IsTruncStore = true;
 }
 def truncstorei8 : PatFrag<(ops node:$val, node:$ptr),
                            (truncstore node:$val, node:$ptr)> {
   let IsStore = true;
   let MemoryVT = i8;
+  let IsTruncStore = true;
 }
 def truncstorei16 : PatFrag<(ops node:$val, node:$ptr),
                             (truncstore node:$val, node:$ptr)> {
   let IsStore = true;
   let MemoryVT = i16;
+  let IsTruncStore = true;
 }
 def truncstorei32 : PatFrag<(ops node:$val, node:$ptr),
                             (truncstore node:$val, node:$ptr)> {
   let IsStore = true;
   let MemoryVT = i32;
+  let IsTruncStore = true;
 }
 def truncstoref16 : PatFrag<(ops node:$val, node:$ptr),
                             (truncstore node:$val, node:$ptr)> {
   let IsStore = true;
   let MemoryVT = f16;
 }
 def truncstoref32 : PatFrag<(ops node:$val, node:$ptr),
                             (truncstore node:$val, node:$ptr)> {

llvm/lib/Target/AArch64/AArch64Combine.td

 def bitfield_extract_from_sext_inreg : GICombineRule<
   (defs root:$root, build_fn_matchinfo:$info),
   (match (wip_match_opcode G_SEXT_INREG):$root,
     [{ return matchBitfieldExtractFromSExtInReg(*${root}, MRI, ${info}); }]),
   (apply [{ return Helper.applyBuildFn(*${root}, ${info}); }])>;
 
 def form_bitfield_extract : GICombineGroup<[bitfield_extract_from_sext_inreg]>;
 
+def form_truncstore_matchdata : GIDefMatchData<"Register">;
+def form_truncstore : GICombineRule<
+  (defs root:$root, form_truncstore_matchdata:$matchinfo),
+  (match (wip_match_opcode G_STORE):$root,
+          [{ return matchFormTruncstore(*${root}, MRI, ${matchinfo}); }]),
+  (apply [{ applyFormTruncstore(*${root}, MRI, B, Observer, ${matchinfo}); }])
+>;
+
 // Post-legalization combines which should happen at all optimization levels.
 // (E.g. ones that facilitate matching for the selector) For example, matching
 // pseudos.
 def AArch64PostLegalizerLoweringHelper
     : GICombinerHelper<"AArch64GenPostLegalizerLoweringHelper",
                        [shuffle_vector_lowering, vashr_vlshr_imm,
-                        icmp_lowering, build_vector_lowering]> {
+                        icmp_lowering, build_vector_lowering,
+                        form_truncstore]> {
   let DisableRuleOption = "aarch64postlegalizerlowering-disable-rule";
 }
 
 // Post-legalization combines which are primarily optimizations.
 def AArch64PostLegalizerCombinerHelper
     : GICombinerHelper<"AArch64GenPostLegalizerCombinerHelper",
                        [copy_prop, erase_undef_store, combines_for_extload,
                         sext_trunc_sextload,
                         hoist_logic_op_with_same_opcode_hands,
                         redundant_and, xor_of_and_with_same_reg,
                         extractvecelt_pairwise_add, redundant_or,
                         mul_const, redundant_sext_inreg,
                         form_bitfield_extract, rotate_out_of_range]> {
   let DisableRuleOption = "aarch64postlegalizercombiner-disable-rule";
 }

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

getActionDefinitionsBuilder(G_LOAD)
       .clampMaxNumElements(0, s8, 16)
       .clampMaxNumElements(0, s16, 8)
       .clampMaxNumElements(0, s32, 4)
       .clampMaxNumElements(0, s64, 2)
       .customIf(IsPtrVecPred);
 
   getActionDefinitionsBuilder(G_STORE)
       .legalForTypesWithMemDesc({{s8, p0, 8, 8},
+                                 {s16, p0, 8, 8}, // truncstorei8 from s16
+                                 {s32, p0, 8, 8}, // truncstorei8 from s32
+                                 {s64, p0, 8, 8}, // truncstorei8 from s64
                                  {s16, p0, 16, 8},
+                                 {s32, p0, 16, 8}, // truncstorei16 from s32
+                                 {s64, p0, 16, 8}, // truncstorei16 from s64
                                  {s32, p0, 8, 8},
                                  {s32, p0, 16, 8},
                                  {s32, p0, 32, 8},
                                  {s64, p0, 64, 8},
+                                 {s64, p0, 32, 8}, // truncstorei32 from s64
                                  {p0, p0, 64, 8},
                                  {s128, p0, 128, 8},
                                  {v16s8, p0, 128, 8},
                                  {v8s8, p0, 64, 8},
                                  {v4s16, p0, 64, 8},
                                  {v8s16, p0, 128, 8},
                                  {v2s32, p0, 64, 8},
                                  {v4s32, p0, 128, 8},

llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp

static bool applySwapICmpOperands(MachineInstr &MI,
   Observer.changedInstr(MI);
   MI.getOperand(1).setPredicate(CmpInst::getSwappedPredicate(Pred));
   MI.getOperand(2).setReg(RHS);
   MI.getOperand(3).setReg(LHS);
   Observer.changedInstr(MI);
   return true;
 }
 
+static bool matchFormTruncstore(MachineInstr &MI, MachineRegisterInfo &MRI,
+                                Register &SrcReg) {
+  assert(MI.getOpcode() == TargetOpcode::G_STORE);
+  Register DstReg = MI.getOperand(0).getReg();
+  if (MRI.getType(DstReg).isVector())
+    return false;
+  // Match a store of a truncate.
+  return mi_match(DstReg, MRI, m_GTrunc(m_Reg(SrcReg)));
+}
+
+static bool applyFormTruncstore(MachineInstr &MI, MachineRegisterInfo &MRI,
+                                MachineIRBuilder &B,
+                                GISelChangeObserver &Observer,
+                                Register &SrcReg) {
+  assert(MI.getOpcode() == TargetOpcode::G_STORE);
+  Observer.changingInstr(MI);
+  MI.getOperand(0).setReg(SrcReg);
+  Observer.changedInstr(MI);
+  return true;
+}
+
 #define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS
 #include "AArch64GenPostLegalizeGILowering.inc"
 #undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS
 
 namespace {
 #define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_H
 #include "AArch64GenPostLegalizeGILowering.inc"
 #undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_H

llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll

 ; CHECK-LSE-O0-NEXT:    ret
    store atomic i32 4, i32* %p seq_cst, align 4
    ret void
 }
 
 define void @atomic_store_relaxed_8(i8* %p, i32 %off32, i8 %val) #0 {
 ; CHECK-NOLSE-O1-LABEL: atomic_store_relaxed_8:
 ; CHECK-NOLSE-O1:       ; %bb.0:
-; CHECK-NOLSE-O1-NEXT:    add x8, x0, w1, sxtw
-; CHECK-NOLSE-O1-NEXT:    sub x9, x0, #256 ; =256
-; CHECK-NOLSE-O1-NEXT:    add x10, x0, #291, lsl #12 ; =1191936
+; CHECK-NOLSE-O1-NEXT:    add x8, x0, #291, lsl #12 ; =1191936
 ; CHECK-NOLSE-O1-NEXT:    strb w2, [x0, #4095]
+; CHECK-NOLSE-O1-NEXT:    strb w2, [x0, w1, sxtw]
+; CHECK-NOLSE-O1-NEXT:    sturb w2, [x0, #-256]
 ; CHECK-NOLSE-O1-NEXT:    strb w2, [x8]
-; CHECK-NOLSE-O1-NEXT:    strb w2, [x9]
-; CHECK-NOLSE-O1-NEXT:    strb w2, [x10]
 ; CHECK-NOLSE-O1-NEXT:    ret
 ;
 ; CHECK-NOLSE-O0-LABEL: atomic_store_relaxed_8:
 ; CHECK-NOLSE-O0:       ; %bb.0:
 ; CHECK-NOLSE-O0-NEXT:    strb w2, [x0, #4095]
-; CHECK-NOLSE-O0-NEXT:    add x8, x0, w1, sxtw
-; CHECK-NOLSE-O0-NEXT:    strb w2, [x8]
-; CHECK-NOLSE-O0-NEXT:    subs x8, x0, #256 ; =256
-; CHECK-NOLSE-O0-NEXT:    strb w2, [x8]
+; CHECK-NOLSE-O0-NEXT:    strb w2, [x0, w1, sxtw]
+; CHECK-NOLSE-O0-NEXT:    sturb w2, [x0, #-256]
 ; CHECK-NOLSE-O0-NEXT:    add x8, x0, #291, lsl #12 ; =1191936
 ; CHECK-NOLSE-O0-NEXT:    strb w2, [x8]
 ; CHECK-NOLSE-O0-NEXT:    ret
 ;
 ; CHECK-LSE-O1-LABEL: atomic_store_relaxed_8:
 ; CHECK-LSE-O1:       ; %bb.0:
 ; CHECK-LSE-O1-NEXT:    strb w2, [x0, #4095]
-; CHECK-LSE-O1-NEXT:    add x8, x0, w1, sxtw
-; CHECK-LSE-O1-NEXT:    strb w2, [x8]
-; CHECK-LSE-O1-NEXT:    sub x8, x0, #256 ; =256
-; CHECK-LSE-O1-NEXT:    strb w2, [x8]
+; CHECK-LSE-O1-NEXT:    strb w2, [x0, w1, sxtw]
+; CHECK-LSE-O1-NEXT:    sturb w2, [x0, #-256]
 ; CHECK-LSE-O1-NEXT:    add x8, x0, #291, lsl #12 ; =1191936
 ; CHECK-LSE-O1-NEXT:    strb w2, [x8]
 ; CHECK-LSE-O1-NEXT:    ret
 ;
 ; CHECK-LSE-O0-LABEL: atomic_store_relaxed_8:
 ; CHECK-LSE-O0:       ; %bb.0:
 ; CHECK-LSE-O0-NEXT:    strb w2, [x0, #4095]
-; CHECK-LSE-O0-NEXT:    add x8, x0, w1, sxtw
-; CHECK-LSE-O0-NEXT:    strb w2, [x8]
-; CHECK-LSE-O0-NEXT:    subs x8, x0, #256 ; =256
-; CHECK-LSE-O0-NEXT:    strb w2, [x8]
+; CHECK-LSE-O0-NEXT:    strb w2, [x0, w1, sxtw]
+; CHECK-LSE-O0-NEXT:    sturb w2, [x0, #-256]
 ; CHECK-LSE-O0-NEXT:    add x8, x0, #291, lsl #12 ; =1191936
 ; CHECK-LSE-O0-NEXT:    strb w2, [x8]
 ; CHECK-LSE-O0-NEXT:    ret
   %ptr_unsigned = getelementptr i8, i8* %p, i32 4095
   store atomic i8 %val, i8* %ptr_unsigned monotonic, align 1
 
   %ptr_regoff = getelementptr i8, i8* %p, i32 %off32
   store atomic i8 %val, i8* %ptr_regoff unordered, align 1
 
   %ptr_unscaled = getelementptr i8, i8* %p, i32 -256
   store atomic i8 %val, i8* %ptr_unscaled monotonic, align 1
 
   %ptr_random = getelementptr i8, i8* %p, i32 1191936 ; 0x123000 (i.e. ADD imm)
   store atomic i8 %val, i8* %ptr_random unordered, align 1
 
   ret void
 }
 
 define void @atomic_store_relaxed_16(i16* %p, i32 %off32, i16 %val) #0 {
 ; CHECK-NOLSE-O1-LABEL: atomic_store_relaxed_16:
 ; CHECK-NOLSE-O1:       ; %bb.0:
-; CHECK-NOLSE-O1-NEXT:    add x8, x0, w1, sxtw #1
-; CHECK-NOLSE-O1-NEXT:    sub x9, x0, #256 ; =256
-; CHECK-NOLSE-O1-NEXT:    add x10, x0, #291, lsl #12 ; =1191936
+; CHECK-NOLSE-O1-NEXT:    add x8, x0, #291, lsl #12 ; =1191936
 ; CHECK-NOLSE-O1-NEXT:    strh w2, [x0, #8190]
+; CHECK-NOLSE-O1-NEXT:    strh w2, [x0, w1, sxtw #1]
+; CHECK-NOLSE-O1-NEXT:    sturh w2, [x0, #-256]
 ; CHECK-NOLSE-O1-NEXT:    strh w2, [x8]
-; CHECK-NOLSE-O1-NEXT:    strh w2, [x9]
-; CHECK-NOLSE-O1-NEXT:    strh w2, [x10]
 ; CHECK-NOLSE-O1-NEXT:    ret
 ;
 ; CHECK-NOLSE-O0-LABEL: atomic_store_relaxed_16:
 ; CHECK-NOLSE-O0:       ; %bb.0:
 ; CHECK-NOLSE-O0-NEXT:    strh w2, [x0, #8190]
-; CHECK-NOLSE-O0-NEXT:    add x8, x0, w1, sxtw #1
-; CHECK-NOLSE-O0-NEXT:    strh w2, [x8]
-; CHECK-NOLSE-O0-NEXT:    subs x8, x0, #256 ; =256
-; CHECK-NOLSE-O0-NEXT:    strh w2, [x8]
+; CHECK-NOLSE-O0-NEXT:    strh w2, [x0, w1, sxtw #1]
+; CHECK-NOLSE-O0-NEXT:    sturh w2, [x0, #-256]
 ; CHECK-NOLSE-O0-NEXT:    add x8, x0, #291, lsl #12 ; =1191936
 ; CHECK-NOLSE-O0-NEXT:    strh w2, [x8]
 ; CHECK-NOLSE-O0-NEXT:    ret
 ;
 ; CHECK-LSE-O1-LABEL: atomic_store_relaxed_16:
 ; CHECK-LSE-O1:       ; %bb.0:
 ; CHECK-LSE-O1-NEXT:    strh w2, [x0, #8190]
-; CHECK-LSE-O1-NEXT:    add x8, x0, w1, sxtw #1
-; CHECK-LSE-O1-NEXT:    strh w2, [x8]
-; CHECK-LSE-O1-NEXT:    sub x8, x0, #256 ; =256
-; CHECK-LSE-O1-NEXT:    strh w2, [x8]
+; CHECK-LSE-O1-NEXT:    strh w2, [x0, w1, sxtw #1]
+; CHECK-LSE-O1-NEXT:    sturh w2, [x0, #-256]
 ; CHECK-LSE-O1-NEXT:    add x8, x0, #291, lsl #12 ; =1191936
 ; CHECK-LSE-O1-NEXT:    strh w2, [x8]
 ; CHECK-LSE-O1-NEXT:    ret
 ;
 ; CHECK-LSE-O0-LABEL: atomic_store_relaxed_16:
 ; CHECK-LSE-O0:       ; %bb.0:
 ; CHECK-LSE-O0-NEXT:    strh w2, [x0, #8190]
-; CHECK-LSE-O0-NEXT:    add x8, x0, w1, sxtw #1
-; CHECK-LSE-O0-NEXT:    strh w2, [x8]
-; CHECK-LSE-O0-NEXT:    subs x8, x0, #256 ; =256
-; CHECK-LSE-O0-NEXT:    strh w2, [x8]
+; CHECK-LSE-O0-NEXT:    strh w2, [x0, w1, sxtw #1]
+; CHECK-LSE-O0-NEXT:    sturh w2, [x0, #-256]
 ; CHECK-LSE-O0-NEXT:    add x8, x0, #291, lsl #12 ; =1191936
 ; CHECK-LSE-O0-NEXT:    strh w2, [x8]
 ; CHECK-LSE-O0-NEXT:    ret
   %ptr_unsigned = getelementptr i16, i16* %p, i32 4095
   store atomic i16 %val, i16* %ptr_unsigned monotonic, align 2
 
   %ptr_regoff = getelementptr i16, i16* %p, i32 %off32
   store atomic i16 %val, i16* %ptr_regoff unordered, align 2

llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir

bb.1:
     ; CHECK: G_STORE [[LOAD]](<2 x s64>), %ptr(p0) :: (store 16, align 32)
     ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD %ptr, [[C]](s64)
     ; CHECK: G_STORE [[LOAD1]](<2 x s64>), [[PTR_ADD1]](p0) :: (store 16 into unknown-address + 16)
     ; CHECK: RET_ReallyLR
     %ptr:_(p0) = COPY $x0
     %val:_(<4 x s64>) = G_LOAD %ptr(p0) :: (load 32)
     G_STORE %val(<4 x s64>), %ptr(p0) :: (store 32)
     RET_ReallyLR
+...
+---
+name:            test_trunc_store
+body: |
+  bb.0:
+    liveins: $x0, $w1
+
+    ; CHECK-LABEL: name: test_trunc_store
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK: %val64:_(s64) = COPY $x2
+    ; CHECK: G_STORE [[COPY1]](s32), [[COPY]](p0) :: (store 1)
+    ; CHECK: G_STORE [[COPY1]](s32), [[COPY]](p0) :: (store 2)
+    ; CHECK: G_STORE %val64(s64), [[COPY]](p0) :: (store 1)
+    ; CHECK: G_STORE %val64(s64), [[COPY]](p0) :: (store 2)
+    ; CHECK: G_STORE %val64(s64), [[COPY]](p0) :: (store 4)
+    %0:_(p0) = COPY $x0
+    %1:_(s32) = COPY $w1
+    %2:_(s8) = G_TRUNC %1(s32)
+    %val64:_(s64) = COPY $x2
+    G_STORE %1(s32), %0(p0) :: (store 1)
+    G_STORE %1(s32), %0(p0) :: (store 2)
+    G_STORE %val64(s64), %0(p0) :: (store 1)
+    G_STORE %val64(s64), %0(p0) :: (store 2)
+    G_STORE %val64(s64), %0(p0) :: (store 4)
+...

llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-truncstore.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-lowering -global-isel -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            truncstore_s8
+legalized:       true
+body: |
+  bb.0.entry:
+    liveins: $x0
+    ; CHECK-LABEL: name: truncstore_s8
+    ; CHECK: %ptr:_(p0) = COPY $x0
+    ; CHECK: %val:_(s32) = COPY $w1
+    ; CHECK: G_STORE %val(s32), %ptr(p0) :: (store 1)
+    %ptr:_(p0) = COPY $x0
+    %val:_(s32) = COPY $w1
+    %trunc:_(s8) = G_TRUNC %val
+    G_STORE %trunc(s8), %ptr(p0) :: (store 1)
+...
+---
+name:            truncstore_vector
+legalized:       true
+body: |
+  bb.0.entry:
+    liveins: $x0
+    ; CHECK-LABEL: name: truncstore_vector
+    ; CHECK: %ptr:_(p0) = COPY $x0
+    ; CHECK: %val:_(<4 x s32>) = COPY $q0
+    ; CHECK: %trunc:_(<4 x s8>) = G_TRUNC %val(<4 x s32>)
+    ; CHECK: G_STORE %trunc(<4 x s8>), %ptr(p0) :: (store 4)
+    %ptr:_(p0) = COPY $x0
+    %val:_(<4 x s32>) = COPY $q0
+    %trunc:_(<4 x s8>) = G_TRUNC %val
+    G_STORE %trunc(<4 x s8>), %ptr(p0) :: (store 4)
+...

llvm/test/CodeGen/AArch64/GlobalISel/select-store.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=aarch64-- -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64-- -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=1 %s -o - | FileCheck %s
 
 --- |
   target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
   define void @store_s64_gpr(i64* %addr) { ret void }
   define void @store_s32_gpr(i32* %addr) { ret void }
   define void @store_s16_gpr(i16* %addr) { ret void }
   define void @store_s8_gpr(i8* %addr) { ret void }
--- |
   define void @store_4xi32(<4 x i32> %v, <4 x i32>* %ptr) { ret void }
   define void @store_8xi16(<8 x i16> %v, <8 x i16>* %ptr) { ret void }
   define void @store_16xi8(<16 x i8> %v, <16 x i8>* %ptr) { ret void }
 
   @x = external hidden local_unnamed_addr global i32*, align 8
   define void @store_adrp_add_low() { ret void }
   define void @store_adrp_add_low_foldable_offset() { ret void }
   define void @store_adrp_add_low_unfoldable_offset() { ret void }
+
+  define void @truncstores(i8* %addr) { ret void }
 ...
 
 ---
 name:            store_s64_gpr
 legalized:       true
 regBankSelected: true
 
 registers:
bb.0:
     ; CHECK: %copy:gpr64all = COPY $x0
     ; CHECK: %add_low:gpr64common = MOVaddr target-flags(aarch64-page) @x + 3, target-flags(aarch64-pageoff, aarch64-nc) @x + 3
     ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY %copy
     ; CHECK: STRXui [[COPY]], %add_low, 0 :: (store 8 into @x)
     %copy:gpr(p0) = COPY $x0
     %adrp:gpr64(p0) = ADRP target-flags(aarch64-page) @x + 3
     %add_low:gpr(p0) = G_ADD_LOW %adrp(p0), target-flags(aarch64-pageoff, aarch64-nc) @x + 3
     G_STORE %copy(p0), %add_low(p0) :: (store 8 into @x)
+...
+
+---
+name:            truncstores
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    liveins: $x0, $w1, $x2
+
+    ; CHECK-LABEL: name: truncstores
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: %val32:gpr32 = COPY $w1
+    ; CHECK: %val64:gpr64 = COPY $x2
+    ; CHECK: STRBBui %val32, [[COPY]], 0 :: (store 1)
+    ; CHECK: STRBBui %val32, [[COPY]], 43 :: (store 1)
+    ; CHECK: STRHHui %val32, [[COPY]], 0 :: (store 2)
+    ; CHECK: STURHHi %val32, [[COPY]], 43 :: (store 2)
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr32 = COPY %val64.sub_32
+    ; CHECK: STRHHui [[COPY1]], [[COPY]], 0 :: (store 2)
+    ; CHECK: [[COPY2:%[0-9]+]]:gpr32 = COPY %val64.sub_32
+    ; CHECK: STURHHi [[COPY2]], [[COPY]], 43 :: (store 2)
+    ; CHECK: [[COPY3:%[0-9]+]]:gpr32 = COPY %val64.sub_32
+    ; CHECK: STRWui [[COPY3]], [[COPY]], 0 :: (store 4)
+    ; CHECK: [[COPY4:%[0-9]+]]:gpr32 = COPY %val64.sub_32
+    ; CHECK: STURWi [[COPY4]], [[COPY]], 43 :: (store 4)
+    %0:gpr(p0) = COPY $x0
+    %val32:gpr(s32) = COPY $w1
+    %val64:gpr(s64) = COPY $x2
+    G_STORE %val32, %0 :: (store 1)
+    ; unscaled offset:
+    %cst:gpr(s64) = G_CONSTANT i64 43
+    %newptr:gpr(p0) = G_PTR_ADD %0, %cst
+    G_STORE %val32, %newptr :: (store 1)
+
+    G_STORE %val32, %0 :: (store 2)
+    ; unscaled offset:
+    G_STORE %val32, %newptr :: (store 2)
+
+    G_STORE %val64, %0 :: (store 2)
+    ; unscaled offset:
+    G_STORE %val64, %newptr :: (store 2)
+
+    G_STORE %val64, %0 :: (store 4)
+    ; unscaled offset:
+    G_STORE %val64, %newptr :: (store 4)
+...

llvm/utils/TableGen/GlobalISelEmitter.cpp

private:
   Optional<const CodeGenRegisterClass *>
   getRegClassFromLeaf(TreePatternNode *Leaf);
 
   /// Return a CodeGenRegisterClass for \p N if one can be found. Return None
   /// otherwise.
   Optional<const CodeGenRegisterClass *>
   inferRegClassFromPattern(TreePatternNode *N);
 
+  /// Return the size of the MemoryVT in this predicate, if possible.
+  Optional<unsigned>
+  getMemSizeBitsFromPredicate(const TreePredicateFn &Predicate);
+
   // Add builtin predicates.
   Expected<InstructionMatcher &>
   addBuiltinPredicates(const Record *SrcGIEquivOrNull,
                        const TreePredicateFn &Predicate,
                        InstructionMatcher &InsnMatcher, bool &HasAddedMatcher);
 
 public:
   /// Takes a sequence of \p Rules and group them based on the predicates
if (Pred->getValueAsString("CondString").empty())
       continue;
     declareSubtargetFeature(Pred);
     M.addRequiredFeature(Pred);
   }
 
   return Error::success();
 }
 
+Optional<unsigned> GlobalISelEmitter::getMemSizeBitsFromPredicate(const TreePredicateFn &Predicate) {
+  Optional<LLTCodeGen> MemTyOrNone =
+      MVTToLLT(getValueType(Predicate.getMemoryVT()));
+
+  if (!MemTyOrNone)
+    return None;
+
+  // Align so unusual types like i1 don't get rounded down.
+  return llvm::alignTo(MemTyOrNone->get().getSizeInBits(), 8);
+}
+
 Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
     const Record *SrcGIEquivOrNull, const TreePredicateFn &Predicate,
     InstructionMatcher &InsnMatcher, bool &HasAddedMatcher) {
   if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
     if (const ListInit *AddrSpaces = Predicate.getAddressSpaces()) {
       SmallVector<unsigned, 4> ParsedAddrSpaces;
 
       for (Init *Val : AddrSpaces->getValues()) {
Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
   if (Predicate.isLoad() && Predicate.isAnyExtLoad()) {
     InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
         0, MemoryVsLLTSizePredicateMatcher::LessThan, 0);
     return InsnMatcher;
   }
 
   if (Predicate.isStore()) {
     if (Predicate.isTruncStore()) {
+      if (Predicate.getMemoryVT() != nullptr) {
         // FIXME: If MemoryVT is set, we end up with 2 checks for the MMO size.
+        auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
+        if (!MemSizeInBits)
+          return failedImport("MemVT could not be converted to LLT");
+
+        InsnMatcher.addPredicate<MemorySizePredicateMatcher>(0, *MemSizeInBits /
+                                                                    8);
+      } else {
         InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
             0, MemoryVsLLTSizePredicateMatcher::LessThan, 0);
+      }
       return InsnMatcher;
     }
     if (Predicate.isNonTruncStore()) {
       // We need to check the sizes match here otherwise we could incorrectly
       // match truncating stores with non-truncating ones.
       InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
           0, MemoryVsLLTSizePredicateMatcher::EqualTo, 0);
     }
if (!SrcGIEquivOrNull->isValueUnset("IfZeroExtend") &&
     return InsnMatcher;
 
   // No check required. G_STORE by itself is a non-extending store.
   if (Predicate.isNonTruncStore())
     return InsnMatcher;
 
   if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
     if (Predicate.getMemoryVT() != nullptr) {
-      Optional<LLTCodeGen> MemTyOrNone =
-          MVTToLLT(getValueType(Predicate.getMemoryVT()));
-
-      if (!MemTyOrNone)
+      auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
+      if (!MemSizeInBits)
         return failedImport("MemVT could not be converted to LLT");
 
-      // MMO's work in bytes so we must take care of unusual types like i1
-      // don't round down.
-      unsigned MemSizeInBits =
-          llvm::alignTo(MemTyOrNone->get().getSizeInBits(), 8);
-
       InsnMatcher.addPredicate<MemorySizePredicateMatcher>(0,
-                                                           MemSizeInBits / 8);
+                                                           *MemSizeInBits / 8);
       return InsnMatcher;
     }
   }
 
   if (Predicate.isLoad() || Predicate.isStore()) {
     // No check required. A G_LOAD/G_STORE is an unindexed load.
     if (Predicate.isUnindexed())
       return InsnMatcher;