Index: llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h =================================================================== --- llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -154,8 +154,14 @@ void applySextTruncSextLoad(MachineInstr &MI); /// Match sext_inreg(load p), imm -> sextload p - bool matchSextInRegOfLoad(MachineInstr &MI, std::tuple &MatchInfo); - void applySextInRegOfLoad(MachineInstr &MI, std::tuple &MatchInfo); + /// assert_sext(load p), imm -> sextload p + /// assert_zext(load p), imm -> zextload p + bool + matchExtLoadFromHintOrSextInReg(MachineInstr &MI, + std::tuple &MatchInfo); + void + applyExtLoadFromHintOrSextInReg(MachineInstr &MI, + std::tuple &MatchInfo); /// Try to combine G_[SU]DIV and G_[SU]REM into a single G_[SU]DIVREM /// when their source operands are identical. Index: llvm/include/llvm/Target/GlobalISel/Combine.td =================================================================== --- llvm/include/llvm/Target/GlobalISel/Combine.td +++ llvm/include/llvm/Target/GlobalISel/Combine.td @@ -118,6 +118,7 @@ def apint_matchinfo : GIDefMatchData<"APInt">; def build_fn_matchinfo : GIDefMatchData<"std::function">; +def register_and_unsigned_matchinfo: GIDefMatchData<"std::tuple">; def copy_prop : GICombineRule< (defs root:$d), @@ -125,12 +126,20 @@ [{ return Helper.matchCombineCopy(*${mi}); }]), (apply [{ Helper.applyCombineCopy(*${mi}); }])>; +// Same combine as sext_inreg_of_load, but for hint instructions. This should +// run at all opt levels. +def hint_to_extload : GICombineRule< + (defs root:$root, register_and_unsigned_matchinfo:$matchinfo), + (match (wip_match_opcode G_ASSERT_SEXT, G_ASSERT_ZEXT):$root, + [{ return Helper.matchExtLoadFromHintOrSextInReg(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyExtLoadFromHintOrSextInReg(*${root}, ${matchinfo}); }])>; + def extending_loads : GICombineRule< (defs root:$root, extending_load_matchdata:$matchinfo), (match (wip_match_opcode G_LOAD, G_SEXTLOAD, G_ZEXTLOAD):$root, [{ return Helper.matchCombineExtendingLoads(*${root}, ${matchinfo}); }]), (apply [{ Helper.applyCombineExtendingLoads(*${root}, ${matchinfo}); }])>; -def combines_for_extload: GICombineGroup<[extending_loads]>; +def combines_for_extload: GICombineGroup<[extending_loads, hint_to_extload]>; def sext_trunc_sextload : GICombineRule< (defs root:$d), @@ -138,12 +147,11 @@ [{ return Helper.matchSextTruncSextLoad(*${d}); }]), (apply [{ Helper.applySextTruncSextLoad(*${d}); }])>; -def sext_inreg_of_load_matchdata : GIDefMatchData<"std::tuple">; def sext_inreg_of_load : GICombineRule< - (defs root:$root, sext_inreg_of_load_matchdata:$matchinfo), + (defs root:$root, register_and_unsigned_matchinfo:$matchinfo), (match (wip_match_opcode G_SEXT_INREG):$root, - [{ return Helper.matchSextInRegOfLoad(*${root}, ${matchinfo}); }]), - (apply [{ Helper.applySextInRegOfLoad(*${root}, ${matchinfo}); }])>; + [{ return Helper.matchExtLoadFromHintOrSextInReg(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyExtLoadFromHintOrSextInReg(*${root}, ${matchinfo}); }])>; def combine_indexed_load_store : GICombineRule< (defs root:$root, indexed_load_store_matchdata:$matchinfo), Index: llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp =================================================================== --- llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -684,9 +684,12 @@ MI.eraseFromParent(); } -bool CombinerHelper::matchSextInRegOfLoad( +bool CombinerHelper::matchExtLoadFromHintOrSextInReg( MachineInstr &MI, std::tuple &MatchInfo) { - assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG); + unsigned Opc = MI.getOpcode(); + assert(Opc == TargetOpcode::G_SEXT_INREG || + Opc == TargetOpcode::G_ASSERT_SEXT || + Opc == TargetOpcode::G_ASSERT_ZEXT); // Only supports scalars for now. if (MRI.getType(MI.getOperand(0).getReg()).isVector()) @@ -715,9 +718,12 @@ return true; } -void CombinerHelper::applySextInRegOfLoad( +void CombinerHelper::applyExtLoadFromHintOrSextInReg( MachineInstr &MI, std::tuple &MatchInfo) { - assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG); + unsigned Opc = MI.getOpcode(); + assert(Opc == TargetOpcode::G_SEXT_INREG || + Opc == TargetOpcode::G_ASSERT_SEXT || + Opc == TargetOpcode::G_ASSERT_ZEXT); Register LoadReg; unsigned ScalarSizeBits; std::tie(LoadReg, ScalarSizeBits) = MatchInfo; @@ -734,7 +740,10 @@ auto &MF = Builder.getMF(); auto PtrInfo = MMO.getPointerInfo(); auto *NewMMO = MF.getMachineMemOperand(&MMO, PtrInfo, ScalarSizeBits / 8); - Builder.buildLoadInstr(TargetOpcode::G_SEXTLOAD, MI.getOperand(0).getReg(), + unsigned LoadOpc = Opc == TargetOpcode::G_ASSERT_ZEXT + ? TargetOpcode::G_ZEXTLOAD + : TargetOpcode::G_SEXTLOAD; + Builder.buildLoadInstr(LoadOpc, MI.getOperand(0).getReg(), LoadDef->getPointerReg(), *NewMMO); MI.eraseFromParent(); } Index: llvm/test/CodeGen/AArch64/GlobalISel/combine-hint-extload.mir =================================================================== --- /dev/null +++ llvm/test/CodeGen/AArch64/GlobalISel/combine-hint-extload.mir @@ -0,0 +1,153 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple aarch64 -debugify-and-strip-all-safe -O0 -run-pass=aarch64-prelegalizer-combiner --aarch64prelegalizercombinerhelper-only-enable-rule="hint_to_extload" -global-isel -verify-machineinstrs %s -o - | FileCheck %s +# REQUIRES: asserts +... +--- +name: assert_sext_sextload +tracksRegLiveness: true +body: | + bb.0: + liveins: $x0 + ; CHECK-LABEL: name: assert_sext_sextload + ; CHECK: liveins: $x0 + ; CHECK: %ptr:_(p0) = COPY $x0 + ; CHECK: %hint:_(s32) = G_SEXTLOAD %ptr(p0) :: (load (s8)) + ; CHECK: $w0 = COPY %hint(s32) + ; CHECK: RET_ReallyLR implicit $w0 + %ptr:_(p0) = COPY $x0 + %load:_(s32) = G_LOAD %ptr(p0) :: (load (s8)) + %hint:_(s32) = G_ASSERT_SEXT %load, 8 + $w0 = COPY %hint(s32) + RET_ReallyLR implicit $w0 +... +--- +name: assert_zext_zextload +tracksRegLiveness: true +body: | + bb.0: + liveins: $x0 + ; CHECK-LABEL: name: assert_zext_zextload + ; CHECK: liveins: $x0 + ; CHECK: %ptr:_(p0) = COPY $x0 + ; CHECK: %hint:_(s32) = G_ZEXTLOAD %ptr(p0) :: (load (s8)) + ; CHECK: $w0 = COPY %hint(s32) + ; CHECK: RET_ReallyLR implicit $w0 + %ptr:_(p0) = COPY $x0 + %load:_(s32) = G_LOAD %ptr(p0) :: (load (s8)) + %hint:_(s32) = G_ASSERT_ZEXT %load, 8 + $w0 = COPY %hint(s32) + RET_ReallyLR implicit $w0 +... +--- +name: assert_sext_sextload_hint_smaller +tracksRegLiveness: true +body: | + bb.0: + liveins: $x0 + ; CHECK-LABEL: name: assert_sext_sextload_hint_smaller + ; CHECK: liveins: $x0 + ; CHECK: %ptr:_(p0) = COPY $x0 + ; CHECK: %hint:_(s32) = G_SEXTLOAD %ptr(p0) :: (load (s8), align 2) + ; CHECK: $w0 = COPY %hint(s32) + ; CHECK: RET_ReallyLR implicit $w0 + %ptr:_(p0) = COPY $x0 + %load:_(s32) = G_LOAD %ptr(p0) :: (load (s16)) + %hint:_(s32) = G_ASSERT_SEXT %load, 8 + $w0 = COPY %hint(s32) + RET_ReallyLR implicit $w0 +... +--- +name: assert_sext_sextload_memop_smaller +tracksRegLiveness: true +body: | + bb.0: + liveins: $x0 + ; CHECK-LABEL: name: assert_sext_sextload_memop_smaller + ; CHECK: liveins: $x0 + ; CHECK: %ptr:_(p0) = COPY $x0 + ; CHECK: %hint:_(s32) = G_SEXTLOAD %ptr(p0) :: (load (s8)) + ; CHECK: $w0 = COPY %hint(s32) + ; CHECK: RET_ReallyLR implicit $w0 + %ptr:_(p0) = COPY $x0 + %load:_(s32) = G_LOAD %ptr(p0) :: (load (s8)) + %hint:_(s32) = G_ASSERT_SEXT %load, 16 + $w0 = COPY %hint(s32) + RET_ReallyLR implicit $w0 +... +--- +name: hint_imm_too_small +tracksRegLiveness: true +body: | + bb.0: + liveins: $x0 + ; CHECK-LABEL: name: hint_imm_too_small + ; CHECK: liveins: $x0 + ; CHECK: %ptr:_(p0) = COPY $x0 + ; CHECK: %load:_(s32) = G_LOAD %ptr(p0) :: (load (s8)) + ; CHECK: %hint:_(s32) = G_ASSERT_SEXT %load, 4 + ; CHECK: $w0 = COPY %hint(s32) + ; CHECK: RET_ReallyLR implicit $w0 + %ptr:_(p0) = COPY $x0 + %load:_(s32) = G_LOAD %ptr(p0) :: (load (s8)) + %hint:_(s32) = G_ASSERT_SEXT %load, 4 + $w0 = COPY %hint(s32) + RET_ReallyLR implicit $w0 +... +--- +name: hint_imm_non_pow_2 +tracksRegLiveness: true +body: | + bb.0: + liveins: $x0 + ; CHECK-LABEL: name: hint_imm_non_pow_2 + ; CHECK: liveins: $x0 + ; CHECK: %ptr:_(p0) = COPY $x0 + ; CHECK: %load:_(s32) = G_LOAD %ptr(p0) :: (load (s16)) + ; CHECK: %hint:_(s32) = G_ASSERT_SEXT %load, 9 + ; CHECK: $w0 = COPY %hint(s32) + ; CHECK: RET_ReallyLR implicit $w0 + %ptr:_(p0) = COPY $x0 + %load:_(s32) = G_LOAD %ptr(p0) :: (load (s16)) + %hint:_(s32) = G_ASSERT_SEXT %load, 9 + $w0 = COPY %hint(s32) + RET_ReallyLR implicit $w0 +... +--- +name: more_than_one_use +tracksRegLiveness: true +body: | + bb.0: + liveins: $x0 + ; CHECK-LABEL: name: more_than_one_use + ; CHECK: liveins: $x0 + ; CHECK: %ptr:_(p0) = COPY $x0 + ; CHECK: %load:_(s32) = G_LOAD %ptr(p0) :: (load (s8)) + ; CHECK: %hint:_(s32) = G_ASSERT_SEXT %load, 8 + ; CHECK: %add:_(s32) = G_ADD %hint, %load + ; CHECK: $w0 = COPY %add(s32) + ; CHECK: RET_ReallyLR implicit $w0 + %ptr:_(p0) = COPY $x0 + %load:_(s32) = G_LOAD %ptr(p0) :: (load (s8)) + %hint:_(s32) = G_ASSERT_SEXT %load, 8 + %add:_(s32) = G_ADD %hint, %load + $w0 = COPY %add(s32) + RET_ReallyLR implicit $w0 +... +--- +name: not_simple +tracksRegLiveness: true +body: | + bb.0: + liveins: $x0 + ; CHECK-LABEL: name: not_simple + ; CHECK: liveins: $x0 + ; CHECK: %ptr:_(p0) = COPY $x0 + ; CHECK: %load:_(s32) = G_LOAD %ptr(p0) :: (load seq_cst (s8)) + ; CHECK: %hint:_(s32) = G_ASSERT_SEXT %load, 8 + ; CHECK: $w0 = COPY %hint(s32) + ; CHECK: RET_ReallyLR implicit $w0 + %ptr:_(p0) = COPY $x0 + %load:_(s32) = G_LOAD %ptr(p0) :: (load seq_cst (s8)) + %hint:_(s32) = G_ASSERT_SEXT %load, 8 + $w0 = COPY %hint(s32) + RET_ReallyLR implicit $w0