diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -36,6 +36,13 @@ MachineInstr *MI; }; +struct IndexedLoadStoreMatchInfo { + Register Addr; + Register Base; + Register Offset; + bool IsPre; +}; + class CombinerHelper { protected: MachineIRBuilder &Builder; @@ -84,6 +91,8 @@ /// Combine \p MI into a pre-indexed or post-indexed load/store operation if /// legal and the surrounding code makes it useful. bool tryCombineIndexedLoadStore(MachineInstr &MI); + bool matchCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo); + void applyCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo); bool matchElideBrByInvertingCond(MachineInstr &MI); void applyElideBrByInvertingCond(MachineInstr &MI); diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -73,6 +73,7 @@ } def extending_load_matchdata : GIDefMatchData<"PreferredTuple">; +def indexed_load_store_matchdata : GIDefMatchData<"IndexedLoadStoreMatchInfo">; /// The operator at the root of a GICombineRule.Match dag. def match; @@ -106,6 +107,12 @@ (defs root:$root, extending_load_matchdata:$matchinfo), (match [{ return Helper.matchCombineExtendingLoads(${root}, ${matchinfo}); }]), (apply [{ Helper.applyCombineExtendingLoads(${root}, ${matchinfo}); }])>; +def combines_for_extload: GICombineGroup<[extending_loads]>; + +def combine_indexed_load_store : GICombineRule< + (defs root:$root, indexed_load_store_matchdata:$matchinfo), + (match [{ return Helper.matchCombineIndexedLoadStore(${root}, ${matchinfo}); }]), + (apply [{ Helper.applyCombineIndexedLoadStore(${root}, ${matchinfo}); }])>; // FIXME: Is there a reason this wasn't in tryCombine? I've left it out of // all_combines because it wasn't there. @@ -114,6 +121,4 @@ (match [{ return Helper.matchElideBrByInvertingCond(${d}); }]), (apply [{ Helper.applyElideBrByInvertingCond(${d}); }])>; -def combines_for_extload: GICombineGroup<[extending_loads]>; - -def all_combines : GICombineGroup<[trivial_combines, combines_for_extload]>; +def all_combines : GICombineGroup<[trivial_combines, combines_for_extload, combine_indexed_load_store]>; diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -711,18 +711,36 @@ } bool CombinerHelper::tryCombineIndexedLoadStore(MachineInstr &MI) { + IndexedLoadStoreMatchInfo MatchInfo; + if (matchCombineIndexedLoadStore(MI, MatchInfo)) { + applyCombineIndexedLoadStore(MI, MatchInfo); + return true; + } + return false; +} + +bool CombinerHelper::matchCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) { unsigned Opcode = MI.getOpcode(); if (Opcode != TargetOpcode::G_LOAD && Opcode != TargetOpcode::G_SEXTLOAD && Opcode != TargetOpcode::G_ZEXTLOAD && Opcode != TargetOpcode::G_STORE) return false; - bool IsStore = Opcode == TargetOpcode::G_STORE; - Register Addr, Base, Offset; - bool IsPre = findPreIndexCandidate(MI, Addr, Base, Offset); - if (!IsPre && !findPostIndexCandidate(MI, Addr, Base, Offset)) + MatchInfo.IsPre = findPreIndexCandidate(MI, MatchInfo.Addr, MatchInfo.Base, + MatchInfo.Offset); + if (!MatchInfo.IsPre && + !findPostIndexCandidate(MI, MatchInfo.Addr, MatchInfo.Base, + MatchInfo.Offset)) return false; + return true; +} +void CombinerHelper::applyCombineIndexedLoadStore( + MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) { + MachineInstr &AddrDef = *MRI.getUniqueVRegDef(MatchInfo.Addr); + MachineIRBuilder MIRBuilder(MI); + unsigned Opcode = MI.getOpcode(); + bool IsStore = Opcode == TargetOpcode::G_STORE; unsigned NewOpcode; switch (Opcode) { case TargetOpcode::G_LOAD: @@ -741,25 +759,22 @@ llvm_unreachable("Unknown load/store opcode"); } - MachineInstr &AddrDef = *MRI.getUniqueVRegDef(Addr); - MachineIRBuilder MIRBuilder(MI); auto MIB = MIRBuilder.buildInstr(NewOpcode); if (IsStore) { - MIB.addDef(Addr); + MIB.addDef(MatchInfo.Addr); MIB.addUse(MI.getOperand(0).getReg()); } else { MIB.addDef(MI.getOperand(0).getReg()); - MIB.addDef(Addr); + MIB.addDef(MatchInfo.Addr); } - MIB.addUse(Base); - MIB.addUse(Offset); - MIB.addImm(IsPre); + MIB.addUse(MatchInfo.Base); + MIB.addUse(MatchInfo.Offset); + MIB.addImm(MatchInfo.IsPre); MI.eraseFromParent(); AddrDef.eraseFromParent(); LLVM_DEBUG(dbgs() << " Combinined to indexed operation"); - return true; } bool CombinerHelper::matchElideBrByInvertingCond(MachineInstr &MI) { diff --git a/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp --- a/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp +++ b/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp @@ -79,7 +79,7 @@ } } - if (Generated.tryCombineAll(Observer, MI, B)) + if (Generated.tryCombineAll(Observer, MI, B, Helper)) return true; switch (MI.getOpcode()) { @@ -87,11 +87,6 @@ return Helper.tryCombineConcatVectors(MI); case TargetOpcode::G_SHUFFLE_VECTOR: return Helper.tryCombineShuffleVector(MI); - case TargetOpcode::G_LOAD: - case TargetOpcode::G_SEXTLOAD: - case TargetOpcode::G_ZEXTLOAD: - case TargetOpcode::G_STORE: - return Helper.tryCombineIndexedLoadStore(MI); } return false; diff --git a/llvm/utils/TableGen/GICombinerEmitter.cpp b/llvm/utils/TableGen/GICombinerEmitter.cpp --- a/llvm/utils/TableGen/GICombinerEmitter.cpp +++ b/llvm/utils/TableGen/GICombinerEmitter.cpp @@ -710,7 +710,8 @@ << " bool tryCombineAll(\n" << " GISelChangeObserver &Observer,\n" << " MachineInstr &MI,\n" - << " MachineIRBuilder &B) const;\n" + << " MachineIRBuilder &B,\n" + << " CombinerHelper &Helper) const;\n" << "};\n\n"; emitNameMatcher(OS); @@ -766,8 +767,8 @@ OS << "bool " << getClassName() << "::tryCombineAll(\n" << " GISelChangeObserver &Observer,\n" << " MachineInstr &MI,\n" - << " MachineIRBuilder &B) const {\n" - << " CombinerHelper Helper(Observer, B);\n" + << " MachineIRBuilder &B,\n" + << " CombinerHelper &Helper) const {\n" << " MachineBasicBlock *MBB = MI.getParent();\n" << " MachineFunction *MF = MBB->getParent();\n" << " MachineRegisterInfo &MRI = MF->getRegInfo();\n"