Index: llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h =================================================================== --- llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -25,12 +25,14 @@ class GISelChangeObserver; class MachineIRBuilder; +class MachineInstrBuilder; class MachineRegisterInfo; class MachineInstr; class MachineOperand; class GISelKnownBits; class MachineDominatorTree; class LegalizerInfo; +struct LegalityQuery; struct PreferredTuple { LLT Ty; // The result type of the extend. @@ -50,6 +52,26 @@ Register Base; }; +using OperandBuildSteps = + SmallVector, 4>; +struct InstructionBuildSteps { + unsigned Opcode; /// The opcode for the produced instruction. + OperandBuildSteps StepFns; /// Operands to be added to the instruction. + InstructionBuildSteps() {} + InstructionBuildSteps(unsigned Opcode, const OperandBuildSteps &StepFns) + : Opcode(Opcode), StepFns(StepFns) {} +}; + +struct InstructionStepsMatchInfo { + /// Describes instructions to be built during a combine. + /// These are built in order from first to last. + SmallVector InstrsToBuild; + InstructionStepsMatchInfo() {} + InstructionStepsMatchInfo( + std::initializer_list InstrsToBuild) + : InstrsToBuild(InstrsToBuild) {} +}; + class CombinerHelper { protected: MachineIRBuilder &Builder; @@ -69,6 +91,10 @@ return KB; } + /// \return true if the combine is running prior to legalization, or if \p + /// Query is legal on the target. + bool isLegalOrBeforeLegalizer(const LegalityQuery &Query) const; + /// MachineRegisterInfo::replaceRegWith() and inform the observer of the changes void replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, Register ToReg) const; @@ -260,6 +286,15 @@ bool applySimplifyAddToSub(MachineInstr &MI, std::tuple &MatchInfo); + /// Match (logic_op (op x...), (op y...)) -> (op (logic_op x, y)) + bool + matchHoistLogicOpWithSameOpcodeHands(MachineInstr &MI, + InstructionStepsMatchInfo &MatchInfo); + + /// Replace \p MI with a series of instructions described in \p MatchInfo. + bool applyBuildInstructionSteps(MachineInstr &MI, + InstructionStepsMatchInfo &MatchInfo); + /// Try to transform \p MI by using all of the above /// combine functions. Returns true if changed. bool tryCombine(MachineInstr &MI); Index: llvm/include/llvm/Target/GlobalISel/Combine.td =================================================================== --- llvm/include/llvm/Target/GlobalISel/Combine.td +++ llvm/include/llvm/Target/GlobalISel/Combine.td @@ -85,6 +85,7 @@ def extending_load_matchdata : GIDefMatchData<"PreferredTuple">; def indexed_load_store_matchdata : GIDefMatchData<"IndexedLoadStoreMatchInfo">; +def instruction_steps_matchdata: GIDefMatchData<"InstructionStepsMatchInfo">; /// The operator at the root of a GICombineRule.Match dag. def match; @@ -275,6 +276,14 @@ (apply [{ return Helper.applyCombineP2IToI2P(*${root}, ${info}); }]) >; +// Simplify: (logic_op (op x...), (op y...)) -> (op (logic_op x, y)) +def hoist_logic_op_with_same_opcode_hands: GICombineRule < + (defs root:$root, instruction_steps_matchdata:$info), + (match (wip_match_opcode G_AND, G_OR, G_XOR):$root, + [{ return Helper.matchHoistLogicOpWithSameOpcodeHands(*${root}, ${info}); }]), + (apply [{ return Helper.applyBuildInstructionSteps(*${root}, ${info});}]) +>; + // FIXME: These should use the custom predicate feature once it lands. def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero, undef_to_negative_one, @@ -285,10 +294,11 @@ def identity_combines : GICombineGroup<[select_same_val, right_identity_zero, binop_same_val, binop_left_to_zero, - binop_right_to_zero, p2i_to_i2p, + binop_right_to_zero, p2i_to_i2p, i2p_to_p2i]>; def trivial_combines : GICombineGroup<[copy_prop, mul_to_shl]>; def all_combines : GICombineGroup<[trivial_combines, ptr_add_immed_chain, combines_for_extload, combine_indexed_load_store, undef_combines, - identity_combines, simplify_add_to_sub]>; + identity_combines, simplify_add_to_sub, + hoist_logic_op_with_same_opcode_hands]>; Index: llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp =================================================================== --- llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -34,7 +34,6 @@ cl::desc("Force all indexed operations to be " "legal for the GlobalISel combiner")); - CombinerHelper::CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B, GISelKnownBits *KB, MachineDominatorTree *MDT, @@ -44,6 +43,11 @@ (void)this->KB; } +bool CombinerHelper::isLegalOrBeforeLegalizer( + const LegalityQuery &Query) const { + return !LI || LI->getAction(Query).Action == LegalizeActions::Legal; +} + void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, Register ToReg) const { Observer.changingAllUsesOfReg(MRI, FromReg); @@ -1776,6 +1780,109 @@ return true; } +bool CombinerHelper::matchHoistLogicOpWithSameOpcodeHands( + MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) { + // Matches: logic (hand x, ...), (hand y, ...) -> hand (logic x, y), ... + // + // Creates the new hand + logic instruction (but does not insert them.) + // + // On success, MatchInfo is populated with the new instructions. These are + // inserted in applyHoistLogicOpWithSameOpcodeHands. + unsigned LogicOpcode = MI.getOpcode(); + assert(LogicOpcode == TargetOpcode::G_AND || + LogicOpcode == TargetOpcode::G_OR || + LogicOpcode == TargetOpcode::G_XOR); + MachineIRBuilder MIB(MI); + Register Dst = MI.getOperand(0).getReg(); + Register LHSReg = MI.getOperand(1).getReg(); + Register RHSReg = MI.getOperand(2).getReg(); + + // Don't recompute anything. + if (!MRI.hasOneNonDBGUse(LHSReg) || !MRI.hasOneNonDBGUse(RHSReg)) + return false; + + // Make sure we have (hand x, ...), (hand y, ...) + MachineInstr *LeftHandInst = getDefIgnoringCopies(LHSReg, MRI); + MachineInstr *RightHandInst = getDefIgnoringCopies(RHSReg, MRI); + if (!LeftHandInst || !RightHandInst) + return false; + unsigned HandOpcode = LeftHandInst->getOpcode(); + if (HandOpcode != RightHandInst->getOpcode()) + return false; + if (!LeftHandInst->getOperand(1).isReg() || + !RightHandInst->getOperand(1).isReg()) + return false; + + // Make sure the types match up, and if we're doing this post-legalization, + // we end up with legal types. + Register X = LeftHandInst->getOperand(1).getReg(); + Register Y = RightHandInst->getOperand(1).getReg(); + LLT XTy = MRI.getType(X); + LLT YTy = MRI.getType(Y); + if (XTy != YTy) + return false; + if (!isLegalOrBeforeLegalizer({LogicOpcode, {XTy, YTy}})) + return false; + + // Optional extra source register. + Register ExtraHandOpSrcReg; + switch (HandOpcode) { + default: + return false; + case TargetOpcode::G_ANYEXT: + case TargetOpcode::G_SEXT: + case TargetOpcode::G_ZEXT: { + // Match: logic (ext X), (ext Y) --> ext (logic X, Y) + break; + } + case TargetOpcode::G_AND: + case TargetOpcode::G_ASHR: + case TargetOpcode::G_LSHR: + case TargetOpcode::G_SHL: { + // Match: logic (binop x, z), (binop y, z) -> binop (logic x, y), z + MachineOperand &ZOp = LeftHandInst->getOperand(2); + if (!matchEqualDefs(ZOp, RightHandInst->getOperand(2))) + return false; + ExtraHandOpSrcReg = ZOp.getReg(); + break; + } + } + + // Record the steps to build the new instructions. + // + // Steps to build (logic x, y) + auto NewLogicDst = MRI.createGenericVirtualRegister(XTy); + OperandBuildSteps LogicBuildSteps = { + [=](MachineInstrBuilder &MIB) { MIB.addDef(NewLogicDst); }, + [=](MachineInstrBuilder &MIB) { MIB.addReg(X); }, + [=](MachineInstrBuilder &MIB) { MIB.addReg(Y); }}; + InstructionBuildSteps LogicSteps(LogicOpcode, LogicBuildSteps); + + // Steps to build hand (logic x, y), ...z + OperandBuildSteps HandBuildSteps = { + [=](MachineInstrBuilder &MIB) { MIB.addDef(Dst); }, + [=](MachineInstrBuilder &MIB) { MIB.addReg(NewLogicDst); }}; + if (ExtraHandOpSrcReg.isValid()) + HandBuildSteps.push_back( + [=](MachineInstrBuilder &MIB) { MIB.addReg(ExtraHandOpSrcReg); }); + InstructionBuildSteps HandSteps(HandOpcode, HandBuildSteps); + + MatchInfo = InstructionStepsMatchInfo({LogicSteps, HandSteps}); + return true; +} + +bool CombinerHelper::applyBuildInstructionSteps( + MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) { + Builder.setInstr(MI); + for (auto &InstrToBuild : MatchInfo.InstrsToBuild) { + MachineInstrBuilder Instr = Builder.buildInstr(InstrToBuild.Opcode); + for (auto &StepFn : InstrToBuild.StepFns) + StepFn(Instr); + } + MI.eraseFromParent(); + return true; +} + bool CombinerHelper::tryCombine(MachineInstr &MI) { if (tryCombineCopy(MI)) return true; Index: llvm/lib/Target/AArch64/AArch64Combine.td =================================================================== --- llvm/lib/Target/AArch64/AArch64Combine.td +++ llvm/lib/Target/AArch64/AArch64Combine.td @@ -79,6 +79,7 @@ def AArch64PostLegalizerCombinerHelper : GICombinerHelper<"AArch64GenPostLegalizerCombinerHelper", [erase_undef_store, combines_for_extload, - sext_trunc_sextload, shuffle_vector_pseudos]> { + sext_trunc_sextload, shuffle_vector_pseudos, + hoist_logic_op_with_same_opcode_hands]> { let DisableRuleOption = "aarch64postlegalizercombiner-disable-rule"; }