diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -62,6 +62,9 @@ bool matchCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo); void applyCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo); + bool matchCombineBr(MachineInstr &MI); + bool tryCombineBr(MachineInstr &MI); + /// Try to transform \p MI by using all of the above /// combine functions. Returns true if changed. bool tryCombine(MachineInstr &MI); diff --git a/llvm/include/llvm/CodeGen/MachineOperand.h b/llvm/include/llvm/CodeGen/MachineOperand.h --- a/llvm/include/llvm/CodeGen/MachineOperand.h +++ b/llvm/include/llvm/CodeGen/MachineOperand.h @@ -684,6 +684,11 @@ Contents.RegMask = RegMaskPtr; } + void setPredicate(unsigned Predicate) { + assert(isPredicate() && "Wrong MachineOperand mutator"); + Contents.Pred = Predicate; + } + //===--------------------------------------------------------------------===// // Other methods. //===--------------------------------------------------------------------===// diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -342,6 +342,68 @@ Observer.changedInstr(MI); } +bool CombinerHelper::matchCombineBr(MachineInstr &MI) { + assert(MI.getOpcode() == TargetOpcode::G_BR && "Expected a G_BR"); + // Try to match the following: + // bb1: + // %c(s32) = G_ICMP pred, %a, %b + // %c1(s1) = G_TRUNC %c(s32) + // G_BRCOND %c1, %bb2 + // G_BR %bb3 + // bb2: + // ... + // bb3: + + // The above pattern does not have a fall through to the successor bb2, always + // resulting in a branch no matter which path is taken. Here we try to find + // and replace that pattern with conditional branch to bb3 and otherwise + // fallthrough to bb2. + + MachineBasicBlock *MBB = MI.getParent(); + MachineBasicBlock::iterator BrIt(MI); + if (BrIt == MBB->begin()) + return false; + assert(std::next(BrIt) == MBB->end() && "expected G_BR to be a terminator"); + + MachineInstr *BrCond = &*std::prev(BrIt); + if (BrCond->getOpcode() != TargetOpcode::G_BRCOND) + return false; + + // Check that the next block is the conditional branch target. + if (!MBB->isLayoutSuccessor(BrCond->getOperand(1).getMBB())) + return false; + + MachineInstr *CmpMI = MRI.getVRegDef(BrCond->getOperand(0).getReg()); + if (!CmpMI || CmpMI->getOpcode() != TargetOpcode::G_ICMP || + !MRI.hasOneUse(CmpMI->getOperand(0).getReg())) + return false; + return true; +} + +bool CombinerHelper::tryCombineBr(MachineInstr &MI) { + if (!matchCombineBr(MI)) + return false; + MachineBasicBlock *BrTarget = MI.getOperand(0).getMBB(); + MachineBasicBlock::iterator BrIt(MI); + MachineInstr *BrCond = &*std::prev(BrIt); + MachineInstr *CmpMI = MRI.getVRegDef(BrCond->getOperand(0).getReg()); + + CmpInst::Predicate InversePred = CmpInst::getInversePredicate( + (CmpInst::Predicate)CmpMI->getOperand(1).getPredicate()); + + // Invert the G_ICMP condition. + Observer.changingInstr(*CmpMI); + CmpMI->getOperand(1).setPredicate(InversePred); + Observer.changedInstr(*CmpMI); + + // Change the conditional branch target. + Observer.changingInstr(*BrCond); + BrCond->getOperand(1).setMBB(BrTarget); + Observer.changedInstr(*BrCond); + MI.eraseFromParent(); + return true; +} + bool CombinerHelper::tryCombine(MachineInstr &MI) { if (tryCombineCopy(MI)) return true; diff --git a/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp --- a/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp +++ b/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp @@ -45,6 +45,8 @@ return false; case TargetOpcode::COPY: return Helper.tryCombineCopy(MI); + case TargetOpcode::G_BR: + return Helper.tryCombineBr(MI); case TargetOpcode::G_LOAD: case TargetOpcode::G_SEXTLOAD: case TargetOpcode::G_ZEXTLOAD: diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/localizer-arm64-tti.ll b/llvm/test/CodeGen/AArch64/GlobalISel/localizer-arm64-tti.ll --- a/llvm/test/CodeGen/AArch64/GlobalISel/localizer-arm64-tti.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/localizer-arm64-tti.ll @@ -26,10 +26,9 @@ ; CHECK: [[GV2:%[0-9]+]]:gpr(p0) = G_GLOBAL_VALUE @var3 ; CHECK: [[C3:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 0 ; CHECK: [[LOAD:%[0-9]+]]:gpr(s32) = G_LOAD [[GV]](p0) :: (load 4 from @var1) - ; CHECK: [[ICMP:%[0-9]+]]:gpr(s32) = G_ICMP intpred(eq), [[LOAD]](s32), [[C]] + ; CHECK: [[ICMP:%[0-9]+]]:gpr(s32) = G_ICMP intpred(ne), [[LOAD]](s32), [[C]] ; CHECK: [[TRUNC:%[0-9]+]]:gpr(s1) = G_TRUNC [[ICMP]](s32) - ; CHECK: G_BRCOND [[TRUNC]](s1), %bb.2 - ; CHECK: G_BR %bb.3 + ; CHECK: G_BRCOND [[TRUNC]](s1), %bb.3 ; CHECK: bb.2.if.then: ; CHECK: successors: %bb.3(0x80000000) ; CHECK: [[GV3:%[0-9]+]]:gpr(p0) = G_GLOBAL_VALUE @var2 diff --git a/llvm/test/CodeGen/AArch64/speculation-hardening.ll b/llvm/test/CodeGen/AArch64/speculation-hardening.ll --- a/llvm/test/CodeGen/AArch64/speculation-hardening.ll +++ b/llvm/test/CodeGen/AArch64/speculation-hardening.ll @@ -26,12 +26,11 @@ ; NOSLH-NOT: csetm x16, ne %cmp = icmp slt i32 %call, %N br i1 %cmp, label %if.then, label %return -; GlobalISel lowers the branch to a b.ne sometimes instead of b.ge as expected.. -; CHECK: b.[[COND:(ge)|(lt)|(ne)]] +; CHECK: b.[[COND:(ge)|(lt)|(ne)|(eq)]] if.then: ; preds = %entry -; NOSLH-NOT: csel x16, x16, xzr, {{(lt)|(ge)|(eq)}} -; SLH-DAG: csel x16, x16, xzr, {{(lt)|(ge)|(eq)}} +; NOSLH-NOT: csel x16, x16, xzr, {{(lt)|(ge)|(eq)|(ne)}} +; SLH-DAG: csel x16, x16, xzr, {{(lt)|(ge)|(eq)|(ne)}} %idxprom = sext i32 %i to i64 %arrayidx = getelementptr inbounds i8, i8* %p, i64 %idxprom %0 = load i8, i8* %arrayidx, align 1 @@ -136,7 +135,7 @@ %l7 = icmp sgt i32 %l0, %l1 br i1 %l7, label %then, label %else ; GlobalISel lowers the branch to a b.ne sometimes instead of b.ge as expected.. -; CHECK: b.[[COND:(le)|(gt)|(ne)]] +; CHECK: b.[[COND:(le)|(gt)|(ne)|(eq)]] then: ; SLH-DAG: csel x16, x16, xzr, [[COND]] @@ -144,7 +143,7 @@ br label %postif else: -; SLH-DAG: csel x16, x16, xzr, {{(gt)|(le)|(eq)}} +; SLH-DAG: csel x16, x16, xzr, {{(gt)|(le)|(eq)|(ne)}} %l11 = sdiv i32 %l1, %l0 br label %postif