diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -336,8 +336,8 @@ bool matchAndWithTrivialMask(MachineInstr &MI, Register &Replacement); /// Combine inverting a result of a compare into the opposite cond code. - bool matchNotCmp(MachineInstr &MI, Register &CmpReg); - bool applyNotCmp(MachineInstr &MI, Register &CmpReg); + bool matchNotCmp(MachineInstr &MI, SmallVector &RegsToNegate); + bool applyNotCmp(MachineInstr &MI, SmallVector &RegsToNegate); /// Try to transform \p MI by using all of the above /// combine functions. Returns true if changed. diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -326,7 +326,7 @@ (apply [{ return Helper.replaceSingleDefInstWithReg(*${root}, ${matchinfo}); }]) >; -def not_cmp_fold_matchinfo : GIDefMatchData<"Register">; +def not_cmp_fold_matchinfo : GIDefMatchData<"SmallVector">; def not_cmp_fold : GICombineRule< (defs root:$d, not_cmp_fold_matchinfo:$info), (match (wip_match_opcode G_XOR): $d, diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -2135,7 +2135,8 @@ return KB->maskedValueIsZero(Replacement, ~Mask); } -bool CombinerHelper::matchNotCmp(MachineInstr &MI, Register &CmpReg) { +bool CombinerHelper::matchNotCmp(MachineInstr &MI, + SmallVector &RegsToNegate) { assert(MI.getOpcode() == TargetOpcode::G_XOR); Register XorSrc; int64_t Cst; @@ -2150,33 +2151,68 @@ if (!isConstTrueVal(TLI, Cst) && !(Ty == LLT::scalar(1) && Cst == -1)) return false; - // Now try match src to either icmp or fcmp. - if (!mi_match(XorSrc, MRI, m_GICmp(m_Pred(), m_Reg(), m_Reg()))) { - // Try fcmp. - if (!mi_match(XorSrc, MRI, m_GFCmp(m_Pred(), m_Reg(), m_Reg()))) + // Check that XorSrc is the root of a tree of comparisons combined with ANDs + // and ORs. The suffix of RegsToNegate starting from index I is used a work + // list of tree nodes to visit. + RegsToNegate.push_back(XorSrc); + for (unsigned I = 0; I < RegsToNegate.size(); ++I) { + Register Reg = RegsToNegate[I]; + if (!MRI.hasOneNonDBGUse(Reg)) return false; + MachineInstr *Def = MRI.getVRegDef(Reg); + switch (Def->getOpcode()) { + default: + return false; + case TargetOpcode::G_ICMP: + case TargetOpcode::G_FCMP: + // When we apply the combine we will invert the predicate. + break; + case TargetOpcode::G_AND: + case TargetOpcode::G_OR: + // Implement De Morgan's laws: + // ~(x & y) -> ~x | ~y + // ~(x | y) -> ~x & ~y + // When we apply the combine we will change the opcode... + if (!Def->getOperand(1).isReg() || !Def->getOperand(2).isReg()) + return false; + // ... and negate the operands. + RegsToNegate.push_back(Def->getOperand(1).getReg()); + RegsToNegate.push_back(Def->getOperand(2).getReg()); + break; + } } - if (!MRI.hasOneNonDBGUse(XorSrc)) - return false; - CmpReg = XorSrc; return true; } -bool CombinerHelper::applyNotCmp(MachineInstr &MI, Register &CmpReg) { - MachineInstr *CmpDef = MRI.getVRegDef(CmpReg); - assert(CmpDef && "Should have been given an MI reg"); - assert(CmpDef->getOpcode() == TargetOpcode::G_ICMP || - CmpDef->getOpcode() == TargetOpcode::G_FCMP); - - Observer.changingInstr(*CmpDef); - MachineOperand &PredOp = CmpDef->getOperand(1); - CmpInst::Predicate NewP = CmpInst::getInversePredicate( - (CmpInst::Predicate)PredOp.getPredicate()); - PredOp.setPredicate(NewP); - Observer.changedInstr(*CmpDef); +bool CombinerHelper::applyNotCmp(MachineInstr &MI, + SmallVector &RegsToNegate) { + for (Register Reg : RegsToNegate) { + MachineInstr *Def = MRI.getVRegDef(Reg); + Observer.changingInstr(*Def); + // For each comparison, invert the opcode. For each AND and OR, change the + // opcode. + switch (Def->getOpcode()) { + default: + llvm_unreachable("Unexpected opcode"); + case TargetOpcode::G_ICMP: + case TargetOpcode::G_FCMP: { + MachineOperand &PredOp = Def->getOperand(1); + CmpInst::Predicate NewP = CmpInst::getInversePredicate( + (CmpInst::Predicate)PredOp.getPredicate()); + PredOp.setPredicate(NewP); + break; + } + case TargetOpcode::G_AND: + Def->setDesc(Builder.getTII().get(TargetOpcode::G_OR)); + break; + case TargetOpcode::G_OR: + Def->setDesc(Builder.getTII().get(TargetOpcode::G_AND)); + break; + } + Observer.changedInstr(*Def); + } - replaceRegWith(MRI, MI.getOperand(0).getReg(), - CmpDef->getOperand(0).getReg()); + replaceRegWith(MRI, MI.getOperand(0).getReg(), MI.getOperand(1).getReg()); MI.eraseFromParent(); return true; }