diff --git a/llvm/lib/Transforms/Scalar/DivRemPairs.cpp b/llvm/lib/Transforms/Scalar/DivRemPairs.cpp --- a/llvm/lib/Transforms/Scalar/DivRemPairs.cpp +++ b/llvm/lib/Transforms/Scalar/DivRemPairs.cpp @@ -238,8 +238,52 @@ if (!DivDominates && !DT.dominates(RemInst, DivInst)) { // We have matching div-rem pair, but they are in two different blocks, // neither of which dominates one another. - // FIXME: We could hoist both ops to the common predecessor block? - continue; + + BasicBlock *PredBB = nullptr; + BasicBlock *DivBB = DivInst->getParent(); + BasicBlock *RemBB = RemInst->getParent(); + + // It's only safe to hoist if every instruction before the Div/Rem in the + // basic block is guaranteed to transfer execution. + auto IsSafeToHoist = [](Instruction *DivOrRem, BasicBlock *ParentBB) { + for (auto I = ParentBB->begin(), E = DivOrRem->getIterator(); I != E; + ++I) + if (!isGuaranteedToTransferExecutionToSuccessor(&*I)) + return false; + + return true; + }; + + // Look for something like this + // PredBB + // | \ + // | Rem + // | / + // Div + // + // If the Rem block has a single predecessor and successor, and all paths + // from PredBB go to either RemBB or DivBB, and execution of RemBB and + // DivBB will always reach the Div/Rem, we can hoist Div to PredBB. If + // we have a DivRem operation we can also hoist Rem. Otherwise we'll leave + // Rem where it is and rewrite it to mul/sub. + // FIXME: We could handle more hoisting cases. + if (RemBB->getSingleSuccessor() == DivBB) + PredBB = RemBB->getUniquePredecessor(); + + if (PredBB && IsSafeToHoist(RemInst, RemBB) && + IsSafeToHoist(DivInst, DivBB) && + llvm::all_of(successors(PredBB), [&](BasicBlock *BB) { + return BB == DivBB || BB == RemBB; + })) { + DivDominates = true; + DivInst->moveBefore(PredBB->getTerminator()); + Changed = true; + if (HasDivRemOp) { + RemInst->moveBefore(PredBB->getTerminator()); + continue; + } + } else + continue; } // The target does not have a single div/rem operation, diff --git a/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll b/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll --- a/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll +++ b/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll @@ -210,13 +210,13 @@ ; CHECK-LABEL: @remainder_triangle_i64( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[CMP:%.*]] = icmp ne i64* [[RP:%.*]], null +; CHECK-NEXT: [[DIV:%.*]] = udiv i64 [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: [[REM:%.*]] = urem i64 [[A]], [[B]] ; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[END:%.*]] ; CHECK: if.then: -; CHECK-NEXT: [[REM:%.*]] = urem i64 [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: store i64 [[REM]], i64* [[RP]], align 4 ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[DIV:%.*]] = udiv i64 [[A]], [[B]] ; CHECK-NEXT: ret i64 [[DIV]] ; entry: @@ -239,13 +239,16 @@ ; CHECK-LABEL: @remainder_triangle_i128( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[CMP:%.*]] = icmp ne i128* [[RP:%.*]], null +; CHECK-NEXT: [[A_FROZEN:%.*]] = freeze i128 [[A:%.*]] +; CHECK-NEXT: [[B_FROZEN:%.*]] = freeze i128 [[B:%.*]] +; CHECK-NEXT: [[DIV:%.*]] = udiv i128 [[A_FROZEN]], [[B_FROZEN]] ; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[END:%.*]] ; CHECK: if.then: -; CHECK-NEXT: [[REM:%.*]] = urem i128 [[A:%.*]], [[B:%.*]] -; CHECK-NEXT: store i128 [[REM]], i128* [[RP]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = mul i128 [[DIV]], [[B_FROZEN]] +; CHECK-NEXT: [[REM_DECOMPOSED:%.*]] = sub i128 [[A_FROZEN]], [[TMP0]] +; CHECK-NEXT: store i128 [[REM_DECOMPOSED]], i128* [[RP]], align 4 ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[DIV:%.*]] = udiv i128 [[A]], [[B]] ; CHECK-NEXT: ret i128 [[DIV]] ; entry: @@ -265,16 +268,16 @@ define i64 @remainder_triangle_i64_multiple_rem_edges(i64 %a, i64 %b, i64 %c, i64* %rp) { ; CHECK-LABEL: @remainder_triangle_i64_multiple_rem_edges( ; CHECK-NEXT: entry: +; CHECK-NEXT: [[DIV:%.*]] = udiv i64 [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: [[REM:%.*]] = urem i64 [[A]], [[B]] ; CHECK-NEXT: switch i64 [[C:%.*]], label [[SW_DEFAULT:%.*]] [ ; CHECK-NEXT: i64 0, label [[SW_BB:%.*]] ; CHECK-NEXT: i64 2, label [[SW_BB]] ; CHECK-NEXT: ] ; CHECK: sw.bb: -; CHECK-NEXT: [[REM:%.*]] = urem i64 [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: store i64 [[REM]], i64* [[RP:%.*]], align 4 ; CHECK-NEXT: br label [[SW_DEFAULT]] ; CHECK: sw.default: -; CHECK-NEXT: [[DIV:%.*]] = udiv i64 [[A]], [[B]] ; CHECK-NEXT: ret i64 [[DIV]] ; entry: @@ -296,16 +299,16 @@ define i64 @remainder_triangle_i64_multiple_div_edges(i64 %a, i64 %b, i64 %c, i64* %rp) { ; CHECK-LABEL: @remainder_triangle_i64_multiple_div_edges( ; CHECK-NEXT: entry: +; CHECK-NEXT: [[DIV:%.*]] = udiv i64 [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: [[REM:%.*]] = urem i64 [[A]], [[B]] ; CHECK-NEXT: switch i64 [[C:%.*]], label [[SW_DEFAULT:%.*]] [ ; CHECK-NEXT: i64 0, label [[SW_BB:%.*]] ; CHECK-NEXT: i64 2, label [[SW_BB]] ; CHECK-NEXT: ] ; CHECK: sw.default: -; CHECK-NEXT: [[REM:%.*]] = urem i64 [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: store i64 [[REM]], i64* [[RP:%.*]], align 4 ; CHECK-NEXT: br label [[SW_BB]] ; CHECK: sw.bb: -; CHECK-NEXT: [[DIV:%.*]] = udiv i64 [[A]], [[B]] ; CHECK-NEXT: ret i64 [[DIV]] ; entry: