Index: llvm/include/llvm/Transforms/Utils/GuardUtils.h =================================================================== --- llvm/include/llvm/Transforms/Utils/GuardUtils.h +++ llvm/include/llvm/Transforms/Utils/GuardUtils.h @@ -17,6 +17,7 @@ class BranchInst; class CallInst; class Function; +class Instruction; class Value; /// Splits control flow at point of \p Guard, replacing it with explicit branch @@ -34,10 +35,9 @@ /// path. Branch remains widenable after transform. void widenWidenableBranch(BranchInst *WidenableBR, Value *NewCond); -/// Given a branch we know is widenable (defined per Analysis/GuardUtils.h), -/// *set* it's condition such that (only) 'Cond' is known to hold on the taken -/// path and that the branch remains widenable after transform. -void setWidenableBranchCond(BranchInst *WidenableBR, Value *Cond); +/// Widen \p WidenableCondition with a \p NewCond by replacing its use with a +/// 'WidenableCondition and NewCond' inserted right after \p WidenableCondition. +void widenWidenableCondition(Instruction *WidenableCondition, Value *NewCond); } // llvm Index: llvm/lib/Transforms/Scalar/GuardWidening.cpp =================================================================== --- llvm/lib/Transforms/Scalar/GuardWidening.cpp +++ llvm/lib/Transforms/Scalar/GuardWidening.cpp @@ -80,21 +80,6 @@ namespace { -// Get the condition of \p I. It can either be a guard or a conditional branch. -static Value *getCondition(Instruction *I) { - if (IntrinsicInst *GI = dyn_cast(I)) { - assert(GI->getIntrinsicID() == Intrinsic::experimental_guard && - "Bad guard intrinsic?"); - return GI->getArgOperand(0); - } - Value *Cond, *WC; - BasicBlock *IfTrueBB, *IfFalseBB; - if (parseWidenableBranch(I, Cond, WC, IfTrueBB, IfFalseBB)) - return Cond; - - return cast(I)->getCondition(); -} - // Set the condition for \p I to \p NewCond. \p I can either be a guard or a // conditional branch. static void setCondition(Instruction *I, Value *NewCond) { @@ -182,7 +167,7 @@ static StringRef scoreTypeToString(WideningScore WS); /// Compute the score for widening the condition in \p DominatedInstr - /// into \p DominatingGuard. + /// into \p WideningPoint. WideningScore computeWideningScore(Instruction *DominatedInstr, Instruction *WideningPoint, SmallVectorImpl &ChecksToHoist, @@ -221,8 +206,8 @@ SmallVectorImpl &ChecksToWiden, Instruction *InsertPt, Value *&Result); - Value *hoistChecks(SmallVectorImpl &ChecksToHoist, - Instruction *InsertPt, Value *OldCondition); + void hoistChecks(SmallVectorImpl &ChecksToHoist, + Instruction *InsertPt); /// Adds freeze to Orig and push it as far as possible very aggressively. /// Also replaces all uses of frozen instruction with frozen version. @@ -303,13 +288,10 @@ Instruction *ToWiden) { Value *Result; Instruction *InsertPt = findInsertionPointForWideCondition(ToWiden); - if (!mergeChecks(ChecksToHoist, ChecksToWiden, InsertPt, Result)) - Result = hoistChecks(ChecksToHoist, InsertPt, getCondition(ToWiden)); - if (isGuardAsWidenableBranch(ToWiden)) { - setWidenableBranchCond(cast(ToWiden), Result); - return; - } - setCondition(ToWiden, Result); + if (mergeChecks(ChecksToHoist, ChecksToWiden, InsertPt, Result)) + setCondition(ToWiden, Result); + else + hoistChecks(ChecksToHoist, InsertPt); } public: @@ -355,7 +337,7 @@ assert(EliminatedGuardsAndBranches.empty() || Changed); for (auto *I : EliminatedGuardsAndBranches) if (!WidenedGuards.count(I)) { - assert(isa(getCondition(I)) && "Should be!"); + setCondition(I, ConstantInt::getTrue(I->getContext())); if (isSupportedGuardInstruction(I)) eliminateGuard(I, MSSAU); else { @@ -444,8 +426,6 @@ SmallVector ChecksToWiden; parseWidenableGuard(BestSoFar, ChecksToWiden); widenGuard(ChecksToHoist, ChecksToWiden, BestSoFar); - auto NewGuardCondition = ConstantInt::getTrue(Instr->getContext()); - setCondition(Instr, NewGuardCondition); EliminatedGuardsAndBranches.push_back(Instr); WidenedGuards.insert(BestSoFar); return true; @@ -758,18 +738,21 @@ return false; } -Value *GuardWideningImpl::hoistChecks(SmallVectorImpl &ChecksToHoist, - Instruction *InsertPt, - Value *OldCondition) { +void GuardWideningImpl::hoistChecks(SmallVectorImpl &ChecksToHoist, + Instruction *InsertPt) { assert(!ChecksToHoist.empty()); IRBuilder<> Builder(InsertPt); makeAvailableAt(ChecksToHoist, InsertPt); - makeAvailableAt(OldCondition, InsertPt); Value *Result = Builder.CreateAnd(ChecksToHoist); Result = freezeAndPush(Result, InsertPt); - Result = Builder.CreateAnd(OldCondition, Result); - Result->setName("wide.chk"); - return Result; + if (isGuard(InsertPt)) { + auto OldCondition = InsertPt->getOperand(0); + makeAvailableAt(OldCondition, InsertPt); + Result = Builder.CreateAnd(OldCondition, Result); + Result->setName("wide.chk"); + setCondition(InsertPt, Result); + } else + widenWidenableCondition(InsertPt, Result); } bool GuardWideningImpl::parseRangeChecks( Index: llvm/lib/Transforms/Utils/GuardUtils.cpp =================================================================== --- llvm/lib/Transforms/Utils/GuardUtils.cpp +++ llvm/lib/Transforms/Utils/GuardUtils.cpp @@ -105,22 +105,12 @@ assert(isWidenableBranch(WidenableBR) && "preserve widenabiliy"); } -void llvm::setWidenableBranchCond(BranchInst *WidenableBR, Value *NewCond) { - assert(isWidenableBranch(WidenableBR) && "precondition"); - - Use *C, *WC; - BasicBlock *IfTrueBB, *IfFalseBB; - parseWidenableBranch(WidenableBR, C, WC, IfTrueBB, IfFalseBB); - if (!C) { - // br (wc()), ... form - IRBuilder<> B(WidenableBR); - WidenableBR->setCondition(B.CreateAnd(NewCond, WC->get())); - } else { - // br (wc & C), ... form - Instruction *WCAnd = cast(WidenableBR->getCondition()); - // Condition is only guaranteed to dominate branch - WCAnd->moveBefore(WidenableBR); - C->set(NewCond); - } - assert(isWidenableBranch(WidenableBR) && "preserve widenabiliy"); +void llvm::widenWidenableCondition(Instruction *WidenableCondition, + Value *NewCond) { + assert(isWidenableCondition(WidenableCondition)); + assert(WidenableCondition->hasOneUse()); + auto U = WidenableCondition->user_back(); + IRBuilder<> B(&*std::next(WidenableCondition->getIterator())); + auto WideCheck = B.CreateAnd(NewCond, WidenableCondition, "wide.check"); + U->replaceUsesOfWith(WidenableCondition, WideCheck); } Index: llvm/test/Transforms/GuardWidening/basic_widenable_condition_guards.ll =================================================================== --- llvm/test/Transforms/GuardWidening/basic_widenable_condition_guards.ll +++ llvm/test/Transforms/GuardWidening/basic_widenable_condition_guards.ll @@ -8,9 +8,9 @@ ; CHECK-LABEL: @f_0( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[COND_1_GW_FR:%.*]] = freeze i1 [[COND_1:%.*]] -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1_GW_FR]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_1_GW_FR]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDE_CHECK]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0:![0-9]+]] ; CHECK: deopt: ; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] @@ -52,9 +52,9 @@ ; CHECK-LABEL: @f_1( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[COND_1_GW_FR:%.*]] = freeze i1 [[COND_1:%.*]] -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1_GW_FR]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_1_GW_FR]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDE_CHECK]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] @@ -114,9 +114,9 @@ ; CHECK-NEXT: [[B_GW_FR:%.*]] = freeze i32 [[B:%.*]] ; CHECK-NEXT: [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 10 ; CHECK-NEXT: [[COND_1:%.*]] = icmp ult i32 [[B_GW_FR]], 10 -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDE_CHECK]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] @@ -236,9 +236,9 @@ ; CHECK-NEXT: [[B_GW_FR:%.*]] = freeze i32 [[B:%.*]] ; CHECK-NEXT: [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 10 ; CHECK-NEXT: [[COND_1:%.*]] = icmp ult i32 [[B_GW_FR]], 10 -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDE_CHECK]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] @@ -295,8 +295,8 @@ ; CHECK-NEXT: [[COND_0:%.*]] = icmp ugt i32 [[A:%.*]], 7 ; CHECK-NEXT: [[WIDE_CHK:%.*]] = icmp uge i32 [[A]], 11 ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] -; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]] +; CHECK-NEXT: br i1 [[WIDE_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] ; CHECK-NEXT: ret void @@ -403,9 +403,9 @@ ; CHECK-NEXT: [[A_GW_FR:%.*]] = freeze i32 [[A:%.*]] ; CHECK-NEXT: [[COND_1:%.*]] = load volatile i1, ptr [[COND_BUF:%.*]], align 1 ; CHECK-NEXT: [[COND_3:%.*]] = icmp ult i32 [[A_GW_FR]], 7 -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[COND_1]], [[COND_3]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_3]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_1]], [[WIDE_CHECK]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] @@ -491,9 +491,9 @@ ; CHECK-NEXT: br i1 undef, label [[LOOP]], label [[LEAVE:%.*]] ; CHECK: leave: ; CHECK-NEXT: [[COND_3:%.*]] = icmp ult i32 [[A_GW_FR]], 7 -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[COND_2:%.*]], [[COND_3]] ; CHECK-NEXT: [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND3]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_3]], [[WIDENABLE_COND3]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_2:%.*]], [[WIDE_CHECK]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof [[PROF0]] ; CHECK: deopt2: ; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] @@ -670,9 +670,9 @@ ; CHECK-NEXT: [[COND_1_GW_FR:%.*]] = freeze i1 [[COND_1:%.*]] ; CHECK-NEXT: br label [[OUTER_HEADER:%.*]] ; CHECK: outer_header: -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1_GW_FR]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_1_GW_FR]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDE_CHECK]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] @@ -764,9 +764,9 @@ ; CHECK-NEXT: [[A29:%.*]] = mul i32 [[A28]], [[A28]] ; CHECK-NEXT: [[A30:%.*]] = mul i32 [[A29]], [[A29]] ; CHECK-NEXT: [[COND:%.*]] = trunc i32 [[A30]] to i1 -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 true, [[COND]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 true, [[WIDE_CHECK]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] @@ -840,8 +840,8 @@ ; CHECK-NEXT: [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 14 ; CHECK-NEXT: [[WIDE_CHK:%.*]] = icmp ult i32 [[A]], 10 ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] -; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]] +; CHECK-NEXT: br i1 [[WIDE_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] ; CHECK-NEXT: ret void @@ -1034,9 +1034,9 @@ ; CHECK-LABEL: @swapped_wb( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[COND_1_GW_FR:%.*]] = freeze i1 [[COND_1:%.*]] -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1_GW_FR]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDENABLE_COND]], [[WIDE_CHK]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_1_GW_FR]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHECK]], [[COND_0:%.*]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] @@ -1077,10 +1077,9 @@ ; CHECK-LABEL: @trivial_wb( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[COND_0_GW_FR:%.*]] = freeze i1 [[COND_0:%.*]] -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 true, [[COND_0_GW_FR]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[TMP0:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] -; CHECK-NEXT: br i1 [[TMP0]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_0_GW_FR]], [[WIDENABLE_COND]] +; CHECK-NEXT: br i1 [[WIDE_CHECK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] ; CHECK-NEXT: ret void Index: llvm/test/Transforms/GuardWidening/loop_invariant_widenable_condition.ll =================================================================== --- llvm/test/Transforms/GuardWidening/loop_invariant_widenable_condition.ll +++ llvm/test/Transforms/GuardWidening/loop_invariant_widenable_condition.ll @@ -11,13 +11,12 @@ ; CHECK-NEXT: [[START_GW_FR:%.*]] = freeze i32 [[START:%.*]] ; CHECK-NEXT: [[X_GW_FR:%.*]] = freeze i32 [[X:%.*]] ; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[START_GW_FR]], [[X_GW_FR]] -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 true, [[COND]] ; CHECK-NEXT: [[WC1:%.*]] = call i1 @llvm.experimental.widenable.condition() +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND]], [[WC1]] ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: ; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[START_GW_FR]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ] -; CHECK-NEXT: [[TMP0:%.*]] = and i1 [[WIDE_CHK]], [[WC1]] -; CHECK-NEXT: br i1 [[TMP0]], label [[GUARD_BLOCK:%.*]], label [[EXIT_BY_WC:%.*]] +; CHECK-NEXT: br i1 [[WIDE_CHECK]], label [[GUARD_BLOCK:%.*]], label [[EXIT_BY_WC:%.*]] ; CHECK: exit_by_wc: ; CHECK-NEXT: [[RVAL1:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 [[IV]]) ] ; CHECK-NEXT: ret i32 [[RVAL1]] @@ -132,12 +131,12 @@ ; CHECK-NEXT: [[START_GW_FR:%.*]] = freeze i32 [[START:%.*]] ; CHECK-NEXT: [[X_GW_FR:%.*]] = freeze i32 [[X:%.*]] ; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[START_GW_FR]], [[X_GW_FR]] -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[C:%.*]], [[COND]] ; CHECK-NEXT: [[WC1:%.*]] = call i1 @llvm.experimental.widenable.condition() +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND]], [[WC1]] +; CHECK-NEXT: [[INVARIANT:%.*]] = and i1 [[C:%.*]], [[WIDE_CHECK]] ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: ; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[START_GW_FR]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ] -; CHECK-NEXT: [[INVARIANT:%.*]] = and i1 [[WIDE_CHK]], [[WC1]] ; CHECK-NEXT: br i1 [[INVARIANT]], label [[GUARD_BLOCK:%.*]], label [[EXIT_BY_WC:%.*]] ; CHECK: exit_by_wc: ; CHECK-NEXT: [[RVAL1:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 [[IV]]) ] Index: llvm/test/Transforms/GuardWidening/mixed_guards.ll =================================================================== --- llvm/test/Transforms/GuardWidening/mixed_guards.ll +++ llvm/test/Transforms/GuardWidening/mixed_guards.ll @@ -46,9 +46,9 @@ ; CHECK-LABEL: @test_02( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[COND_1_GW_FR:%.*]] = freeze i1 [[COND_1:%.*]] -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1_GW_FR]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_1_GW_FR]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDE_CHECK]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] Index: llvm/test/Transforms/GuardWidening/profile-based-profitability_explicit.ll =================================================================== --- llvm/test/Transforms/GuardWidening/profile-based-profitability_explicit.ll +++ llvm/test/Transforms/GuardWidening/profile-based-profitability_explicit.ll @@ -7,9 +7,9 @@ ; CHECK-SAME: (i32 [[N:%.*]], i1 [[COND_1:%.*]], i1 [[COND_2:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[COND_2_GW_FR:%.*]] = freeze i1 [[COND_2]] -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[COND_1]], [[COND_2_GW_FR]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_2_GW_FR]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_1]], [[WIDE_CHECK]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0:![0-9]+]] ; CHECK: deopt: ; CHECK-NEXT: [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ] @@ -76,9 +76,9 @@ ; CHECK-SAME: (i32 [[N:%.*]], i1 [[COND_1:%.*]], i1 [[COND_2:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[COND_2_GW_FR:%.*]] = freeze i1 [[COND_2]] -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[COND_1]], [[COND_2_GW_FR]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_2_GW_FR]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_1]], [[WIDE_CHECK]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ] @@ -145,9 +145,9 @@ ; CHECK-SAME: (i32 [[N:%.*]], i1 [[COND_1:%.*]], i1 [[COND_2:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[COND_2_GW_FR:%.*]] = freeze i1 [[COND_2]] -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[COND_1]], [[COND_2_GW_FR]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_2_GW_FR]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_1]], [[WIDE_CHECK]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ] @@ -213,9 +213,9 @@ ; CHECK-SAME: (i32 [[N:%.*]], i1 [[COND_1:%.*]], i1 [[COND_2:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[COND_2_GW_FR:%.*]] = freeze i1 [[COND_2]] -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[COND_1]], [[COND_2_GW_FR]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_2_GW_FR]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_1]], [[WIDE_CHECK]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ] @@ -281,9 +281,9 @@ ; CHECK-SAME: (i32 [[N:%.*]], i1 [[COND_1:%.*]], i1 [[COND_2:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[COND_2_GW_FR:%.*]] = freeze i1 [[COND_2]] -; CHECK-NEXT: [[WIDE_CHK:%.*]] = and i1 [[COND_1]], [[COND_2_GW_FR]] ; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[COND_2_GW_FR]], [[WIDENABLE_COND]] +; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_1]], [[WIDE_CHECK]] ; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof [[PROF0]] ; CHECK: deopt: ; CHECK-NEXT: [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ] Index: llvm/test/Transforms/GuardWidening/two_forms_behavior_consistency.ll =================================================================== --- llvm/test/Transforms/GuardWidening/two_forms_behavior_consistency.ll +++ llvm/test/Transforms/GuardWidening/two_forms_behavior_consistency.ll @@ -52,13 +52,13 @@ ; BRANCH_FORM-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1 ; BRANCH_FORM-NEXT: [[C1:%.*]] = icmp ult i32 [[IV]], [[A]] ; BRANCH_FORM-NEXT: [[C2:%.*]] = icmp ult i32 [[IV]], [[B_GW_FR]] -; BRANCH_FORM-NEXT: [[WIDE_CHK:%.*]] = and i1 [[C1]], [[C2]] ; BRANCH_FORM-NEXT: [[C3:%.*]] = icmp ult i32 [[IV]], [[C_GW_FR]] -; BRANCH_FORM-NEXT: [[WIDE_CHK13:%.*]] = and i1 [[WIDE_CHK]], [[C3]] ; BRANCH_FORM-NEXT: [[C4:%.*]] = icmp ult i32 [[IV]], [[D_GW_FR]] -; BRANCH_FORM-NEXT: [[WIDE_CHK14:%.*]] = and i1 [[WIDE_CHK13]], [[C4]] ; BRANCH_FORM-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; BRANCH_FORM-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK14]], [[WIDENABLE_COND]] +; BRANCH_FORM-NEXT: [[WIDE_CHECK14:%.*]] = and i1 [[C4]], [[WIDENABLE_COND]] +; BRANCH_FORM-NEXT: [[WIDE_CHECK13:%.*]] = and i1 [[C3]], [[WIDE_CHECK14]] +; BRANCH_FORM-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[C2]], [[WIDE_CHECK13]] +; BRANCH_FORM-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[C1]], [[WIDE_CHECK]] ; BRANCH_FORM-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof [[PROF0:![0-9]+]] ; BRANCH_FORM: deopt: ; BRANCH_FORM-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] @@ -87,13 +87,13 @@ ; BRANCH_FORM_LICM-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1 ; BRANCH_FORM_LICM-NEXT: [[C1:%.*]] = icmp ult i32 [[IV]], [[A]] ; BRANCH_FORM_LICM-NEXT: [[C2:%.*]] = icmp ult i32 [[IV]], [[B_GW_FR]] -; BRANCH_FORM_LICM-NEXT: [[WIDE_CHK:%.*]] = and i1 [[C1]], [[C2]] ; BRANCH_FORM_LICM-NEXT: [[C3:%.*]] = icmp ult i32 [[IV]], [[C_GW_FR]] -; BRANCH_FORM_LICM-NEXT: [[WIDE_CHK13:%.*]] = and i1 [[WIDE_CHK]], [[C3]] ; BRANCH_FORM_LICM-NEXT: [[C4:%.*]] = icmp ult i32 [[IV]], [[D_GW_FR]] -; BRANCH_FORM_LICM-NEXT: [[WIDE_CHK14:%.*]] = and i1 [[WIDE_CHK13]], [[C4]] ; BRANCH_FORM_LICM-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition() -; BRANCH_FORM_LICM-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDE_CHK14]], [[WIDENABLE_COND]] +; BRANCH_FORM_LICM-NEXT: [[WIDE_CHECK14:%.*]] = and i1 [[C4]], [[WIDENABLE_COND]] +; BRANCH_FORM_LICM-NEXT: [[WIDE_CHECK13:%.*]] = and i1 [[C3]], [[WIDE_CHECK14]] +; BRANCH_FORM_LICM-NEXT: [[WIDE_CHECK:%.*]] = and i1 [[C2]], [[WIDE_CHECK13]] +; BRANCH_FORM_LICM-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[C1]], [[WIDE_CHECK]] ; BRANCH_FORM_LICM-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof [[PROF0:![0-9]+]] ; BRANCH_FORM_LICM: deopt: ; BRANCH_FORM_LICM-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ] Index: llvm/test/Transforms/GuardWidening/widen-cond-with-operands.ll =================================================================== --- llvm/test/Transforms/GuardWidening/widen-cond-with-operands.ll +++ llvm/test/Transforms/GuardWidening/widen-cond-with-operands.ll @@ -12,9 +12,9 @@ ; CHECK-NEXT: [[WC1:%.*]] = call i1 @llvm.experimental.widenable.condition() ; CHECK-NEXT: [[WC2:%.*]] = call i1 @llvm.experimental.widenable.condition() ; CHECK-NEXT: [[CHECK:%.*]] = icmp ult i32 [[ZERO]], 0 +; CHECK-NEXT: [[C1:%.*]] = and i1 [[CHECK]], [[WC1]] ; CHECK-NEXT: [[C2:%.*]] = and i1 [[CHECK]], [[WC2]] -; CHECK-NEXT: [[C1:%.*]] = and i1 [[WIDE_CHK]], [[WC1]] -; CHECK-NEXT: br i1 [[C1]], label [[BB6:%.*]], label [[BB9:%.*]] +; CHECK-NEXT: br i1 [[WIDE_CHK]], label [[BB6:%.*]], label [[BB9:%.*]] ; CHECK: bb6: ; CHECK-NEXT: br i1 true, label [[BB7:%.*]], label [[BB8:%.*]] ; CHECK: bb7: