diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -336,8 +336,8 @@ bool matchAndWithTrivialMask(MachineInstr &MI, Register &Replacement); /// Combine inverting a result of a compare into the opposite cond code. - bool matchNotCmp(MachineInstr &MI, Register &CmpReg); - bool applyNotCmp(MachineInstr &MI, Register &CmpReg); + bool matchNotCmp(MachineInstr &MI, SmallVector &RegsToNegate); + bool applyNotCmp(MachineInstr &MI, SmallVector &RegsToNegate); /// Try to transform \p MI by using all of the above /// combine functions. Returns true if changed. diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -326,7 +326,7 @@ (apply [{ return Helper.replaceSingleDefInstWithReg(*${root}, ${matchinfo}); }]) >; -def not_cmp_fold_matchinfo : GIDefMatchData<"Register">; +def not_cmp_fold_matchinfo : GIDefMatchData<"SmallVector">; def not_cmp_fold : GICombineRule< (defs root:$d, not_cmp_fold_matchinfo:$info), (match (wip_match_opcode G_XOR): $d, diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -2135,7 +2135,8 @@ return KB->maskedValueIsZero(Replacement, ~Mask); } -bool CombinerHelper::matchNotCmp(MachineInstr &MI, Register &CmpReg) { +bool CombinerHelper::matchNotCmp(MachineInstr &MI, + SmallVector &RegsToNegate) { assert(MI.getOpcode() == TargetOpcode::G_XOR); Register XorSrc; int64_t Cst; @@ -2150,33 +2151,68 @@ if (!isConstTrueVal(TLI, Cst) && !(Ty == LLT::scalar(1) && Cst == -1)) return false; - // Now try match src to either icmp or fcmp. - if (!mi_match(XorSrc, MRI, m_GICmp(m_Pred(), m_Reg(), m_Reg()))) { - // Try fcmp. - if (!mi_match(XorSrc, MRI, m_GFCmp(m_Pred(), m_Reg(), m_Reg()))) + // Check that XorSrc is the root of a tree of comparisons combined with ANDs + // and ORs. The suffix of RegsToNegate starting from index I is used a work + // list of tree nodes to visit. + RegsToNegate.push_back(XorSrc); + for (unsigned I = 0; I < RegsToNegate.size(); ++I) { + Register Reg = RegsToNegate[I]; + if (!MRI.hasOneNonDBGUse(Reg)) return false; + MachineInstr *Def = MRI.getVRegDef(Reg); + switch (Def->getOpcode()) { + default: + // Don't match if the tree contains anything other than ANDs, ORs and + // comparisons. + return false; + case TargetOpcode::G_ICMP: + case TargetOpcode::G_FCMP: + // When we apply the combine we will invert the predicate. + break; + case TargetOpcode::G_AND: + case TargetOpcode::G_OR: + // Implement De Morgan's laws: + // ~(x & y) -> ~x | ~y + // ~(x | y) -> ~x & ~y + // When we apply the combine we will change the opcode and recursively + // negate the operands. + RegsToNegate.push_back(Def->getOperand(1).getReg()); + RegsToNegate.push_back(Def->getOperand(2).getReg()); + break; + } } - if (!MRI.hasOneNonDBGUse(XorSrc)) - return false; - CmpReg = XorSrc; return true; } -bool CombinerHelper::applyNotCmp(MachineInstr &MI, Register &CmpReg) { - MachineInstr *CmpDef = MRI.getVRegDef(CmpReg); - assert(CmpDef && "Should have been given an MI reg"); - assert(CmpDef->getOpcode() == TargetOpcode::G_ICMP || - CmpDef->getOpcode() == TargetOpcode::G_FCMP); - - Observer.changingInstr(*CmpDef); - MachineOperand &PredOp = CmpDef->getOperand(1); - CmpInst::Predicate NewP = CmpInst::getInversePredicate( - (CmpInst::Predicate)PredOp.getPredicate()); - PredOp.setPredicate(NewP); - Observer.changedInstr(*CmpDef); +bool CombinerHelper::applyNotCmp(MachineInstr &MI, + SmallVector &RegsToNegate) { + for (Register Reg : RegsToNegate) { + MachineInstr *Def = MRI.getVRegDef(Reg); + Observer.changingInstr(*Def); + // For each comparison, invert the opcode. For each AND and OR, change the + // opcode. + switch (Def->getOpcode()) { + default: + llvm_unreachable("Unexpected opcode"); + case TargetOpcode::G_ICMP: + case TargetOpcode::G_FCMP: { + MachineOperand &PredOp = Def->getOperand(1); + CmpInst::Predicate NewP = CmpInst::getInversePredicate( + (CmpInst::Predicate)PredOp.getPredicate()); + PredOp.setPredicate(NewP); + break; + } + case TargetOpcode::G_AND: + Def->setDesc(Builder.getTII().get(TargetOpcode::G_OR)); + break; + case TargetOpcode::G_OR: + Def->setDesc(Builder.getTII().get(TargetOpcode::G_AND)); + break; + } + Observer.changedInstr(*Def); + } - replaceRegWith(MRI, MI.getOperand(0).getReg(), - CmpDef->getOperand(0).getReg()); + replaceRegWith(MRI, MI.getOperand(0).getReg(), MI.getOperand(1).getReg()); MI.eraseFromParent(); return true; } diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-invert-cmp.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-invert-cmp.mir --- a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-invert-cmp.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-invert-cmp.mir @@ -134,3 +134,91 @@ $w1 = COPY %other_use_ext RET_ReallyLR implicit $w0 ... +--- +name: icmp_and_icmp +tracksRegLiveness: true +body: | + bb.1: + liveins: $x0 + + ; CHECK-LABEL: name: icmp_and_icmp + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(sle), [[COPY]](s64), [[C]] + ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ule), [[COPY]](s64), [[C]] + ; CHECK: [[OR:%[0-9]+]]:_(s1) = G_OR [[ICMP]], [[ICMP1]] + ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s1) + ; CHECK: $w0 = COPY [[ANYEXT]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %0:_(s64) = COPY $x0 + %1:_(s64) = G_CONSTANT i64 1 + %2:_(s1) = G_CONSTANT i1 1 + %3:_(s1) = G_ICMP intpred(sgt), %0(s64), %1 + %4:_(s1) = G_ICMP intpred(ugt), %0(s64), %1 + %5:_(s1) = G_AND %3, %4 + %6:_(s1) = G_XOR %5, %2 + %7:_(s32) = G_ANYEXT %6 + $w0 = COPY %7(s32) + RET_ReallyLR implicit $w0 +... +--- +name: icmp_or_icmp +tracksRegLiveness: true +body: | + bb.1: + liveins: $x0 + + ; CHECK-LABEL: name: icmp_or_icmp + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(sle), [[COPY]](s64), [[C]] + ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ule), [[COPY]](s64), [[C]] + ; CHECK: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP]], [[ICMP1]] + ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[AND]](s1) + ; CHECK: $w0 = COPY [[ANYEXT]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %0:_(s64) = COPY $x0 + %1:_(s64) = G_CONSTANT i64 1 + %2:_(s1) = G_CONSTANT i1 1 + %3:_(s1) = G_ICMP intpred(sgt), %0(s64), %1 + %4:_(s1) = G_ICMP intpred(ugt), %0(s64), %1 + %5:_(s1) = G_OR %3, %4 + %6:_(s1) = G_XOR %5, %2 + %7:_(s32) = G_ANYEXT %6 + $w0 = COPY %7(s32) + RET_ReallyLR implicit $w0 +... +--- +name: icmp_and_icmp_or_icmp +tracksRegLiveness: true +body: | + bb.1: + liveins: $x0 + + ; CHECK-LABEL: name: icmp_and_icmp_or_icmp + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(sle), [[COPY]](s64), [[C]] + ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ule), [[COPY]](s64), [[C]] + ; CHECK: [[OR:%[0-9]+]]:_(s1) = G_OR [[ICMP]], [[ICMP1]] + ; CHECK: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY]](s64), [[C]] + ; CHECK: [[AND:%[0-9]+]]:_(s1) = G_AND [[OR]], [[ICMP2]] + ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[AND]](s1) + ; CHECK: $w0 = COPY [[ANYEXT]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %0:_(s64) = COPY $x0 + %1:_(s64) = G_CONSTANT i64 1 + %2:_(s1) = G_CONSTANT i1 1 + %3:_(s1) = G_ICMP intpred(sgt), %0(s64), %1 + %4:_(s1) = G_ICMP intpred(ugt), %0(s64), %1 + %5:_(s1) = G_AND %3, %4 + %6:_(s1) = G_ICMP intpred(ne), %0(s64), %1 + %7:_(s1) = G_OR %5, %6 + %8:_(s1) = G_XOR %7, %2 + %9:_(s32) = G_ANYEXT %8 + $w0 = COPY %9(s32) + RET_ReallyLR implicit $w0 +...