diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp --- a/llvm/lib/Transforms/Scalar/LICM.cpp +++ b/llvm/lib/Transforms/Scalar/LICM.cpp @@ -945,6 +945,47 @@ continue; } + // Try to hoist select instruction that match the following and + // %c and %cond is loop invariant: + // Loop: + // %x = add i32 %y, %c + // %z = select i1 %cond, i32 %y, i32 %x + // ==> + // OutOfLoop: + // %a = select i1 %cond, i32 0, i32 %c + // ... + // Loop: + // %z = add i32 %y, %a + if (I.getOpcode() == Instruction::Select) { + using namespace PatternMatch; + Instruction *BO = nullptr; + Value *B = nullptr; + Value *C = I.getOperand(0); + // TODO: Support other binary instructions. + if (match(I.getOperand(1), + m_OneUse(m_Add(m_Specific(I.getOperand(2)), m_Value(B))))) + BO = cast(I.getOperand(1)); + else if (match( + I.getOperand(2), + m_OneUse(m_Add(m_Specific(I.getOperand(1)), m_Value(B))))) + BO = cast(I.getOperand(2)); + + if (BO && B && CurLoop->isLoopInvariant(C) && + CurLoop->isLoopInvariant(B)) { + BasicBlock *HoistBB = CFH.getOrCreateHoistedBlock(BB); + auto *ToAdd = + SelectInst::Create(C, ConstantInt::get(I.getType(), 0), B); + ToAdd->insertBefore(HoistBB->getTerminator()); + BO->setOperand(1, ToAdd); + I.replaceAllUsesWith(BO); + I.eraseFromParent(); + if (SE) { + SE->forgetValue(BO); + SE->forgetValue(&I); + } + } + } + auto IsInvariantStart = [&](Instruction &I) { using namespace PatternMatch; return I.use_empty() && diff --git a/llvm/test/Transforms/LICM/select.ll b/llvm/test/Transforms/LICM/select.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/LICM/select.ll @@ -0,0 +1,144 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -passes=licm -S %s | FileCheck %s +; +define dso_local i32 @add_constant(i32 %x) { +; CHECK-LABEL: @add_constant( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X:%.*]], 0 +; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[TOBOOL_NOT]], i32 0, i32 10 +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.cond.cleanup: +; CHECK-NEXT: [[Y_1_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ] +; CHECK-NEXT: ret i32 [[Y_1_LCSSA]] +; CHECK: for.body: +; CHECK-NEXT: [[I_04:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ] +; CHECK-NEXT: [[Y_03:%.*]] = phi i32 [ 10, [[ENTRY]] ], [ [[ADD]], [[FOR_BODY]] ] +; CHECK-NEXT: [[ADD]] = add nsw i32 [[Y_03]], [[TMP0]] +; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_04]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], 1000 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]] +; +entry: + %tobool.not = icmp eq i32 %x, 0 + br label %for.body + +for.cond.cleanup: ; preds = %for.body + ret i32 %y.1 + +for.body: ; preds = %for.body, %entry + %i.04 = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %y.03 = phi i32 [ 10, %entry ], [ %y.1, %for.body ] + %add = add nsw i32 %y.03, 10 + %y.1 = select i1 %tobool.not, i32 %y.03, i32 %add + %inc = add nuw nsw i32 %i.04, 1 + %exitcond.not = icmp eq i32 %inc, 1000 + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body +} + +define dso_local i32 @add_constant_commute(i32 %x) { +; CHECK-LABEL: @add_constant_commute( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X:%.*]], 0 +; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[TOBOOL_NOT]], i32 0, i32 10 +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.cond.cleanup: +; CHECK-NEXT: [[Y_1_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ] +; CHECK-NEXT: ret i32 [[Y_1_LCSSA]] +; CHECK: for.body: +; CHECK-NEXT: [[I_04:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ] +; CHECK-NEXT: [[Y_03:%.*]] = phi i32 [ 10, [[ENTRY]] ], [ [[ADD]], [[FOR_BODY]] ] +; CHECK-NEXT: [[ADD]] = add nsw i32 [[Y_03]], [[TMP0]] +; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_04]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], 1000 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]] +; +entry: + %tobool.not = icmp eq i32 %x, 0 + br label %for.body + +for.cond.cleanup: ; preds = %for.body + ret i32 %y.1 + +for.body: ; preds = %for.body, %entry + %i.04 = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %y.03 = phi i32 [ 10, %entry ], [ %y.1, %for.body ] + %add = add nsw i32 %y.03, 10 + %y.1 = select i1 %tobool.not, i32 %add, i32 %y.03 + %inc = add nuw nsw i32 %i.04, 1 + %exitcond.not = icmp eq i32 %inc, 1000 + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body +} + +define dso_local i32 @add_loop_invariant_but_non_constant(i32 %x, i32 %xx) { +; CHECK-LABEL: @add_loop_invariant_but_non_constant( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X:%.*]], 0 +; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[TOBOOL_NOT]], i32 0, i32 [[XX:%.*]] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.cond.cleanup: +; CHECK-NEXT: [[Y_1_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ] +; CHECK-NEXT: ret i32 [[Y_1_LCSSA]] +; CHECK: for.body: +; CHECK-NEXT: [[I_04:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ] +; CHECK-NEXT: [[Y_03:%.*]] = phi i32 [ 10, [[ENTRY]] ], [ [[ADD]], [[FOR_BODY]] ] +; CHECK-NEXT: [[ADD]] = add nsw i32 [[Y_03]], [[TMP0]] +; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_04]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], 1000 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]] +; +entry: + %tobool.not = icmp eq i32 %x, 0 + br label %for.body + +for.cond.cleanup: ; preds = %for.body + ret i32 %y.1 + +for.body: ; preds = %for.body, %entry + %i.04 = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %y.03 = phi i32 [ 10, %entry ], [ %y.1, %for.body ] + %add = add nsw i32 %y.03, %xx + %y.1 = select i1 %tobool.not, i32 %add, i32 %y.03 + %inc = add nuw nsw i32 %i.04, 1 + %exitcond.not = icmp eq i32 %inc, 1000 + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body +} + +declare void @use(i32) + +; Negative test, since %add has multiple uses. +; +define dso_local i32 @multiple_uses(i32 %x) { +; CHECK-LABEL: @multiple_uses( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X:%.*]], 0 +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.cond.cleanup: +; CHECK-NEXT: [[Y_1_LCSSA:%.*]] = phi i32 [ [[Y_1:%.*]], [[FOR_BODY]] ] +; CHECK-NEXT: ret i32 [[Y_1_LCSSA]] +; CHECK: for.body: +; CHECK-NEXT: [[I_04:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ] +; CHECK-NEXT: [[Y_03:%.*]] = phi i32 [ 10, [[ENTRY]] ], [ [[Y_1]], [[FOR_BODY]] ] +; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[Y_03]], 10 +; CHECK-NEXT: [[Y_1]] = select i1 [[TOBOOL_NOT]], i32 [[ADD]], i32 [[Y_03]] +; CHECK-NEXT: call void @use(i32 [[ADD]]) +; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_04]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], 1000 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]] +; +entry: + %tobool.not = icmp eq i32 %x, 0 + br label %for.body + +for.cond.cleanup: ; preds = %for.body + ret i32 %y.1 + +for.body: ; preds = %for.body, %entry + %i.04 = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %y.03 = phi i32 [ 10, %entry ], [ %y.1, %for.body ] + %add = add nsw i32 %y.03, 10 + %y.1 = select i1 %tobool.not, i32 %add, i32 %y.03 + call void @use(i32 %add) + %inc = add nuw nsw i32 %i.04, 1 + %exitcond.not = icmp eq i32 %inc, 1000 + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body +}