Index: llvm/lib/CodeGen/CodeGenPrepare.cpp =================================================================== --- llvm/lib/CodeGen/CodeGenPrepare.cpp +++ llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -7885,26 +7885,52 @@ if (FreezeInst *FI = dyn_cast(I)) { // freeze(icmp a, const)) -> icmp (freeze a), const // This helps generate efficient conditional jumps. - Instruction *CmpI = nullptr; + CmpInst *CmpI = nullptr; if (ICmpInst *II = dyn_cast(FI->getOperand(0))) CmpI = II; else if (FCmpInst *F = dyn_cast(FI->getOperand(0))) CmpI = F->getFastMathFlags().none() ? F : nullptr; - if (CmpI && CmpI->hasOneUse()) { + // Check if the Cmp/Freeze can be altered without increasing instructions. A + // Cmp will be sank to the uses by sinkCmpExpression, in which case it is + // worth folding even if the cmp has extra uses. + auto WillBeSunkOrHasOneUse = [&](CmpInst *CmpI, FreezeInst *FI) { + if (CmpI->hasOneUse()) + return true; + // Return false if the sinking of compares is disabled. + if (TLI->hasMultipleConditionRegisters()) + return false; + if (TLI->useSoftFloat() && isa(CmpI)) + return false; + // Check if the cmp node can be sunk, if so will will be duplicating the + // node anyway, fold it here to allow more transforms. + if (FI->getParent() != CmpI->getParent()) + return true; + return any_of(FI->users(), [&](const User *U) { + return cast(U)->getParent() != CmpI->getParent(); + }); + }; + + if (CmpI && WillBeSunkOrHasOneUse(CmpI, FI)) { auto Op0 = CmpI->getOperand(0), Op1 = CmpI->getOperand(1); bool Const0 = isa(Op0) || isa(Op0) || isa(Op0); bool Const1 = isa(Op1) || isa(Op1) || isa(Op1); - if (Const0 || Const1) { - if (!Const0 || !Const1) { - auto *F = new FreezeInst(Const0 ? Op1 : Op0, "", CmpI); - F->takeName(FI); - CmpI->setOperand(Const0 ? 1 : 0, F); - } - FI->replaceAllUsesWith(CmpI); + if (Const0 != Const1) { + auto *F = new FreezeInst(Const0 ? Op1 : Op0, "", CmpI); + F->takeName(FI); + auto *NewCmpI = + CmpInst::Create(CmpI->getOpcode(), CmpI->getPredicate(), + Const0 ? Op0 : F, Const0 ? F : Op1, "", CmpI); + NewCmpI->setDebugLoc(CmpI->getDebugLoc()); + + FI->replaceAllUsesWith(NewCmpI); FI->eraseFromParent(); + if (CmpI->use_empty()) { + salvageDebugInfo(*CmpI); + CmpI->eraseFromParent(); + } return true; } } Index: llvm/test/CodeGen/PowerPC/select-to-branch.mir =================================================================== --- llvm/test/CodeGen/PowerPC/select-to-branch.mir +++ llvm/test/CodeGen/PowerPC/select-to-branch.mir @@ -26,8 +26,8 @@ ; branch. ; CHECK-LABEL: weighted_select2 ; CHECK: %sel.frozen = freeze i32 %a - ; CHECK: %cmp = icmp ne i32 %sel.frozen, 0 - ; CHECK-NEXT: br i1 %cmp, label %select.end, label %select.false, !prof !15 + ; CHECK: %1 = icmp ne i32 %sel.frozen, 0 + ; CHECK-NEXT: br i1 %1, label %select.end, label %select.false, !prof !15 ; CHECK: select.false: ; CHECK-NEXT: br label %select.end ; CHECK: select.end: @@ -44,8 +44,8 @@ ; to branch, here !16 = 1/101, !16 < 1%, so it will convert select to branch. ; CHECK-LABEL: weighted_select3 ; CHECK: %sel.frozen = freeze i32 %a - ; CHECK: %cmp = icmp ne i32 %sel.frozen, 0 - ; CHECK-NEXT: br i1 %cmp, label %select.end, label %select.false, !prof !16 + ; CHECK: %1 = icmp ne i32 %sel.frozen, 0 + ; CHECK-NEXT: br i1 %1, label %select.end, label %select.false, !prof !16 ; CHECK: select.false: ; CHECK-NEXT: br label %select.end ; CHECK: select.end: Index: llvm/test/Transforms/CodeGenPrepare/AArch64/sink-compare-freeze.ll =================================================================== --- llvm/test/Transforms/CodeGenPrepare/AArch64/sink-compare-freeze.ll +++ llvm/test/Transforms/CodeGenPrepare/AArch64/sink-compare-freeze.ll @@ -47,17 +47,19 @@ define i32 @withfreeze(i32 %a, i1 %d) { ; CHECK-LABEL: @withfreeze( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[A:%.*]], 0 -; CHECK-NEXT: [[F:%.*]] = freeze i1 [[C]] +; CHECK-NEXT: [[F:%.*]] = freeze i32 [[A:%.*]] +; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[A]], 0 ; CHECK-NEXT: call void @use(i1 [[C]]) ; CHECK-NEXT: br i1 [[D:%.*]], label [[LABA:%.*]], label [[LABB:%.*]] ; CHECK: laba: -; CHECK-NEXT: br i1 [[F]], label [[BODY:%.*]], label [[LABC:%.*]] +; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[F]], 0 +; CHECK-NEXT: br i1 [[TMP0]], label [[BODY:%.*]], label [[LABC:%.*]] ; CHECK: labb: -; CHECK-NEXT: br i1 [[F]], label [[BODY]], label [[TAIL:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[F]], 0 +; CHECK-NEXT: br i1 [[TMP1]], label [[BODY]], label [[TAIL:%.*]] ; CHECK: labc: -; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[A]], 0 -; CHECK-NEXT: br i1 [[TMP0]], label [[BODY]], label [[TAIL]] +; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[A]], 0 +; CHECK-NEXT: br i1 [[TMP2]], label [[BODY]], label [[TAIL]] ; CHECK: body: ; CHECK-NEXT: [[B:%.*]] = phi i32 [ 0, [[LABA]] ], [ 1, [[LABB]] ], [ 4, [[LABC]] ] ; CHECK-NEXT: br label [[TAIL]] Index: llvm/test/Transforms/CodeGenPrepare/X86/freeze-brcond.ll =================================================================== --- llvm/test/Transforms/CodeGenPrepare/X86/freeze-brcond.ll +++ llvm/test/Transforms/CodeGenPrepare/X86/freeze-brcond.ll @@ -6,8 +6,8 @@ define void @f1(i32 %a) { ; CHECK-LABEL: @f1( ; CHECK-NEXT: [[FR:%.*]] = freeze i32 [[A:%.*]] -; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[FR]], 0 -; CHECK-NEXT: br i1 [[C]], label [[A:%.*]], label [[B:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[FR]], 0 +; CHECK-NEXT: br i1 [[TMP1]], label [[A:%.*]], label [[B:%.*]] ; CHECK: A: ; CHECK-NEXT: call void @g1() ; CHECK-NEXT: ret void @@ -29,8 +29,8 @@ define void @f2(i32 %a) { ; CHECK-LABEL: @f2( ; CHECK-NEXT: [[FR:%.*]] = freeze i32 [[A:%.*]] -; CHECK-NEXT: [[C:%.*]] = icmp eq i32 0, [[FR]] -; CHECK-NEXT: br i1 [[C]], label [[A:%.*]], label [[B:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 0, [[FR]] +; CHECK-NEXT: br i1 [[TMP1]], label [[A:%.*]], label [[B:%.*]] ; CHECK: A: ; CHECK-NEXT: call void @g1() ; CHECK-NEXT: ret void @@ -52,7 +52,8 @@ define void @f3(i32 %a) { ; CHECK-LABEL: @f3( ; CHECK-NEXT: [[C:%.*]] = icmp eq i32 0, 1 -; CHECK-NEXT: br i1 [[C]], label [[A:%.*]], label [[B:%.*]] +; CHECK-NEXT: [[FR:%.*]] = freeze i1 [[C]] +; CHECK-NEXT: br i1 [[FR]], label [[A:%.*]], label [[B:%.*]] ; CHECK: A: ; CHECK-NEXT: call void @g1() ; CHECK-NEXT: ret void @@ -74,8 +75,8 @@ define i1 @ptrcmp(i8* %p) { ; CHECK-LABEL: @ptrcmp( ; CHECK-NEXT: [[FR:%.*]] = freeze i8* [[P:%.*]] -; CHECK-NEXT: [[C:%.*]] = icmp eq i8* [[FR]], null -; CHECK-NEXT: ret i1 [[C]] +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i8* [[FR]], null +; CHECK-NEXT: ret i1 [[TMP1]] ; %c = icmp eq i8* %p, null %fr = freeze i1 %c @@ -86,8 +87,8 @@ define i1 @fcmp(float %a) { ; CHECK-LABEL: @fcmp( ; CHECK-NEXT: [[FR:%.*]] = freeze float [[A:%.*]] -; CHECK-NEXT: [[C:%.*]] = fcmp oeq float [[FR]], 0.000000e+00 -; CHECK-NEXT: ret i1 [[C]] +; CHECK-NEXT: [[TMP1:%.*]] = fcmp oeq float [[FR]], 0.000000e+00 +; CHECK-NEXT: ret i1 [[TMP1]] ; %c = fcmp oeq float %a, 0.0 %fr = freeze i1 %c @@ -109,8 +110,8 @@ ; CHECK-LABEL: @and_bitmask( ; CHECK-NEXT: [[V:%.*]] = and i32 [[FLAG:%.*]], 1 ; CHECK-NEXT: [[FR:%.*]] = freeze i32 [[V]] -; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[FR]], 0 -; CHECK-NEXT: br i1 [[C]], label [[A:%.*]], label [[B:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[FR]], 0 +; CHECK-NEXT: br i1 [[TMP1]], label [[A:%.*]], label [[B:%.*]] ; CHECK: A: ; CHECK-NEXT: call void @g1() ; CHECK-NEXT: ret void @@ -134,8 +135,8 @@ ; CHECK-LABEL: @and_bitmask_r( ; CHECK-NEXT: [[V:%.*]] = and i32 1, [[FLAG:%.*]] ; CHECK-NEXT: [[FR:%.*]] = freeze i32 [[V]] -; CHECK-NEXT: [[C:%.*]] = icmp eq i32 0, [[FR]] -; CHECK-NEXT: br i1 [[C]], label [[A:%.*]], label [[B:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 0, [[FR]] +; CHECK-NEXT: br i1 [[TMP1]], label [[A:%.*]], label [[B:%.*]] ; CHECK: A: ; CHECK-NEXT: call void @g1() ; CHECK-NEXT: ret void Index: llvm/test/Transforms/CodeGenPrepare/X86/select.ll =================================================================== --- llvm/test/Transforms/CodeGenPrepare/X86/select.ll +++ llvm/test/Transforms/CodeGenPrepare/X86/select.ll @@ -38,8 +38,8 @@ ; CHECK-LABEL: @fdiv_true_sink( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SEL_FROZEN:%.*]] = freeze float [[A:%.*]] -; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt float [[SEL_FROZEN]], 1.000000e+00 -; CHECK-NEXT: br i1 [[CMP]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]] +; CHECK-NEXT: [[TMP0:%.*]] = fcmp ogt float [[SEL_FROZEN]], 1.000000e+00 +; CHECK-NEXT: br i1 [[TMP0]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]] ; CHECK: select.true.sink: ; CHECK-NEXT: [[DIV:%.*]] = fdiv float [[A]], [[B:%.*]] ; CHECK-NEXT: br label [[SELECT_END]] @@ -50,9 +50,9 @@ ; DEBUG-LABEL: @fdiv_true_sink( ; DEBUG-NEXT: entry: ; DEBUG-NEXT: [[SEL_FROZEN:%.*]] = freeze float [[A:%.*]] -; DEBUG-NEXT: [[CMP:%.*]] = fcmp ogt float [[SEL_FROZEN]], 1.000000e+00, !dbg [[DBG24:![0-9]+]] -; DEBUG-NEXT: call void @llvm.dbg.value(metadata i1 [[CMP]], metadata [[META22:![0-9]+]], metadata !DIExpression()), !dbg [[DBG24]] -; DEBUG-NEXT: br i1 [[CMP]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]], !dbg [[DBG25:![0-9]+]] +; DEBUG-NEXT: [[TMP0:%.*]] = fcmp ogt float [[SEL_FROZEN]], 1.000000e+00, !dbg [[DBG24:![0-9]+]] +; DEBUG-NEXT: call void @llvm.dbg.value(metadata i1 undef, metadata [[META22:![0-9]+]], metadata !DIExpression()), !dbg [[DBG24]] +; DEBUG-NEXT: br i1 [[TMP0]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]], !dbg [[DBG25:![0-9]+]] ; DEBUG: select.true.sink: ; DEBUG-NEXT: [[DIV:%.*]] = fdiv float [[A]], [[B:%.*]], !dbg [[DBG26:![0-9]+]] ; DEBUG-NEXT: call void @llvm.dbg.value(metadata float [[DIV]], metadata [[META21:![0-9]+]], metadata !DIExpression()), !dbg [[DBG26]] @@ -73,8 +73,8 @@ ; CHECK-LABEL: @fdiv_false_sink( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SEL_FROZEN:%.*]] = freeze float [[A:%.*]] -; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt float [[SEL_FROZEN]], 3.000000e+00 -; CHECK-NEXT: br i1 [[CMP]], label [[SELECT_END:%.*]], label [[SELECT_FALSE_SINK:%.*]] +; CHECK-NEXT: [[TMP0:%.*]] = fcmp ogt float [[SEL_FROZEN]], 3.000000e+00 +; CHECK-NEXT: br i1 [[TMP0]], label [[SELECT_END:%.*]], label [[SELECT_FALSE_SINK:%.*]] ; CHECK: select.false.sink: ; CHECK-NEXT: [[DIV:%.*]] = fdiv float [[A]], [[B:%.*]] ; CHECK-NEXT: br label [[SELECT_END]] @@ -85,9 +85,9 @@ ; DEBUG-LABEL: @fdiv_false_sink( ; DEBUG-NEXT: entry: ; DEBUG-NEXT: [[SEL_FROZEN:%.*]] = freeze float [[A:%.*]] -; DEBUG-NEXT: [[CMP:%.*]] = fcmp ogt float [[SEL_FROZEN]], 3.000000e+00, !dbg [[DBG33:![0-9]+]] -; DEBUG-NEXT: call void @llvm.dbg.value(metadata i1 [[CMP]], metadata [[META31:![0-9]+]], metadata !DIExpression()), !dbg [[DBG33]] -; DEBUG-NEXT: br i1 [[CMP]], label [[SELECT_END:%.*]], label [[SELECT_FALSE_SINK:%.*]], !dbg [[DBG34:![0-9]+]] +; DEBUG-NEXT: [[TMP0:%.*]] = fcmp ogt float [[SEL_FROZEN]], 3.000000e+00, !dbg [[DBG33:![0-9]+]] +; DEBUG-NEXT: call void @llvm.dbg.value(metadata i1 undef, metadata [[META31:![0-9]+]], metadata !DIExpression()), !dbg [[DBG33]] +; DEBUG-NEXT: br i1 [[TMP0]], label [[SELECT_END:%.*]], label [[SELECT_FALSE_SINK:%.*]], !dbg [[DBG34:![0-9]+]] ; DEBUG: select.false.sink: ; DEBUG-NEXT: [[DIV:%.*]] = fdiv float [[A]], [[B:%.*]], !dbg [[DBG35:![0-9]+]] ; DEBUG-NEXT: call void @llvm.dbg.value(metadata float [[DIV]], metadata [[META30:![0-9]+]], metadata !DIExpression()), !dbg [[DBG35]] @@ -108,8 +108,8 @@ ; CHECK-LABEL: @fdiv_both_sink( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SEL_FROZEN:%.*]] = freeze float [[A:%.*]] -; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt float [[SEL_FROZEN]], 5.000000e+00 -; CHECK-NEXT: br i1 [[CMP]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_FALSE_SINK:%.*]] +; CHECK-NEXT: [[TMP0:%.*]] = fcmp ogt float [[SEL_FROZEN]], 5.000000e+00 +; CHECK-NEXT: br i1 [[TMP0]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_FALSE_SINK:%.*]] ; CHECK: select.true.sink: ; CHECK-NEXT: [[DIV1:%.*]] = fdiv float [[A]], [[B:%.*]] ; CHECK-NEXT: br label [[SELECT_END:%.*]] @@ -123,9 +123,9 @@ ; DEBUG-LABEL: @fdiv_both_sink( ; DEBUG-NEXT: entry: ; DEBUG-NEXT: [[SEL_FROZEN:%.*]] = freeze float [[A:%.*]] -; DEBUG-NEXT: [[CMP:%.*]] = fcmp ogt float [[SEL_FROZEN]], 5.000000e+00, !dbg [[DBG43:![0-9]+]] -; DEBUG-NEXT: call void @llvm.dbg.value(metadata i1 [[CMP]], metadata [[META41:![0-9]+]], metadata !DIExpression()), !dbg [[DBG43]] -; DEBUG-NEXT: br i1 [[CMP]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_FALSE_SINK:%.*]], !dbg [[DBG44:![0-9]+]] +; DEBUG-NEXT: [[TMP0:%.*]] = fcmp ogt float [[SEL_FROZEN]], 5.000000e+00, !dbg [[DBG43:![0-9]+]] +; DEBUG-NEXT: call void @llvm.dbg.value(metadata i1 undef, metadata [[META41:![0-9]+]], metadata !DIExpression()), !dbg [[DBG43]] +; DEBUG-NEXT: br i1 [[TMP0]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_FALSE_SINK:%.*]], !dbg [[DBG44:![0-9]+]] ; DEBUG: select.true.sink: ; DEBUG-NEXT: [[DIV1:%.*]] = fdiv float [[A]], [[B:%.*]], !dbg [[DBG45:![0-9]+]] ; DEBUG-NEXT: call void @llvm.dbg.value(metadata float [[DIV1]], metadata [[META39:![0-9]+]], metadata !DIExpression()), !dbg [[DBG45]]