diff --git a/llvm/lib/Transforms/Scalar/DivRemPairs.cpp b/llvm/lib/Transforms/Scalar/DivRemPairs.cpp --- a/llvm/lib/Transforms/Scalar/DivRemPairs.cpp +++ b/llvm/lib/Transforms/Scalar/DivRemPairs.cpp @@ -238,8 +238,51 @@ if (!DivDominates && !DT.dominates(RemInst, DivInst)) { // We have matching div-rem pair, but they are in two different blocks, // neither of which dominates one another. - // FIXME: We could hoist both ops to the common predecessor block? - continue; + + BasicBlock *PredBB = nullptr; + BasicBlock *DivBB = DivInst->getParent(); + BasicBlock *RemBB = RemInst->getParent(); + + // It's only safe to hoist if every instruction before the Div/Rem in the + // basic block is gauranteed to transfer execution. + auto IsSafeToHoist = [](Instruction *DivOrRem, BasicBlock *ParentBB) { + for (auto I = ParentBB->begin(), E = DivOrRem->getIterator(); + I != E; ++I) + if (!isGuaranteedToTransferExecutionToSuccessor(&*I)) + return false; + + return true; + }; + + // Look for something like this + // PredBB + // | \ + // | Rem + // | / + // Div + // + // If the Rem block has a single predecessor and successor, and all paths + // from PredBB go to either Rem or Div, we can hoist Div to PredBB. If + // we have a DivRem operation we can also hoist Rem. Otherwise we'll leave + // Rem where it is and rewrite it to mul/sub. + // FIXME: We could handle more cases hoisting cases? + if (RemBB->getSingleSuccessor() == DivBB && + RemBB->getSinglePredecessor() != nullptr && + IsSafeToHoist(RemInst, RemBB) && IsSafeToHoist(DivInst, DivBB)) + PredBB = RemInst->getParent()->getSinglePredecessor(); + + // Make sure all paths from PredBB go to either the RemBB or the DivBB. + if (PredBB && llvm::all_of(successors(PredBB), [&](BasicBlock *BB) { + return BB == DivBB || BB == RemBB; })) { + DivDominates = true; + DivInst->moveBefore(PredBB->getTerminator()); + if (HasDivRemOp) { + RemInst->moveBefore(PredBB->getTerminator()); + Changed = true; + continue; + } + } else + continue; } // The target does not have a single div/rem operation, diff --git a/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll b/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll --- a/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll +++ b/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll @@ -204,3 +204,60 @@ %ret = phi i32 [ %t2, %if.then ], [ %t3, %if.else ] ret i32 %ret } + +define i64 @remainder_triangle_i64(i64 %a, i64 %b, i64* %rp) { +; CHECK-LABEL: @remainder_triangle_i64( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CMP:%.*]] = icmp ne i64* [[RP:%.*]], null +; CHECK-NEXT: [[DIV:%.*]] = udiv i64 [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: [[REM:%.*]] = urem i64 [[A]], [[B]] +; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: store i64 [[REM]], i64* [[RP]], align 4 +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: ret i64 [[DIV]] +; +entry: + %cmp = icmp ne i64* %rp, null + br i1 %cmp, label %if.then, label %end + +if.then: + %rem = urem i64 %a, %b + store i64 %rem, i64* %rp + br label %end + +end: + %div = udiv i64 %a, %b + ret i64 %div +} + +define i128 @remainder_triangle_i128(i128 %a, i128 %b, i128* %rp) { +; CHECK-LABEL: @remainder_triangle_i128( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CMP:%.*]] = icmp ne i128* [[RP:%.*]], null +; CHECK-NEXT: [[A_FROZEN:%.*]] = freeze i128 [[A:%.*]] +; CHECK-NEXT: [[B_FROZEN:%.*]] = freeze i128 [[B:%.*]] +; CHECK-NEXT: [[DIV:%.*]] = udiv i128 [[A_FROZEN]], [[B_FROZEN]] +; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[TMP0:%.*]] = mul i128 [[DIV]], [[B_FROZEN]] +; CHECK-NEXT: [[REM_DECOMPOSED:%.*]] = sub i128 [[A_FROZEN]], [[TMP0]] +; CHECK-NEXT: store i128 [[REM_DECOMPOSED]], i128* [[RP]], align 4 +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: ret i128 [[DIV]] +; +entry: + %cmp = icmp ne i128* %rp, null + br i1 %cmp, label %if.then, label %end + +if.then: + %rem = urem i128 %a, %b + store i128 %rem, i128* %rp + br label %end + +end: + %div = udiv i128 %a, %b + ret i128 %div +}