diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp --- a/llvm/lib/Transforms/Scalar/GVN.cpp +++ b/llvm/lib/Transforms/Scalar/GVN.cpp @@ -1222,9 +1222,42 @@ auto *L1 = FindDominatingLoad(Sel->getOperand(1)); auto *L2 = FindDominatingLoad(Sel->getOperand(2)); + + // Try to find an existing value-select, which provides the same value as + // loading from the pointer-select Sel. + if (L1) { + SelectInst *ValueSel = nullptr; + for (User *U : L1->users()) { + ValueSel = dyn_cast(U); + if (ValueSel && ValueSel->getOperand(0) == Sel->getOperand(0) && + ValueSel->getOperand(1) == L1) + break; + ValueSel = nullptr; + } + if (ValueSel && DT.dominates(ValueSel, Sel)) { + auto *Phi0 = dyn_cast(ValueSel->getOperand(2)); + auto *Phi1 = dyn_cast(Sel->getOperand(2)); + if (Phi0 && Phi1 && Phi0->getParent() == Phi1->getParent() && + all_of(Phi0->blocks(), [Phi0, Phi1, Sel, ValueSel](BasicBlock *BB) { + if (Phi0->getIncomingValueForBlock(BB) == ValueSel && + Phi1->getIncomingValueForBlock(BB) == Sel) + return true; + + auto *IncPtr = Phi1->getIncomingValueForBlock(BB); + auto *Load = dyn_cast(Phi0->getIncomingValueForBlock(BB)); + if (Load && Load->getPointerOperand() == IncPtr) { + return Load->getParent()->getSingleSuccessor() == + Sel->getParent() && + Load->getNextNode() == Load->getParent()->getTerminator(); + } + return false; + })) { + return AvailableValueInBlock::get(Sel->getParent(), ValueSel); + } + } + } if (!L1 || !L2) return None; - return AvailableValueInBlock::getSelect(Sel->getParent(), cast(Address)); } @@ -1707,8 +1740,10 @@ // If we had a phi translation failure, we'll have a single entry which is a // clobber in the current block. Reject this early. - if (NumDeps == 1 && - !Deps[0].getResult().isDef() && !Deps[0].getResult().isClobber()) { + if (NumDeps == 1 && !Deps[0].getResult().isDef() && + !Deps[0].getResult().isClobber() && + !tryToConvertLoadOfPtrSelect(Deps[0].getBB(), Deps[0].getAddress(), + getDominatorTree())) { LLVM_DEBUG(dbgs() << "GVN: non-local load "; Load->printAsOperand(dbgs()); dbgs() << " has unknown dependencies\n";); return false; diff --git a/llvm/test/Transforms/GVN/PRE/pre-loop-load-through-select.ll b/llvm/test/Transforms/GVN/PRE/pre-loop-load-through-select.ll --- a/llvm/test/Transforms/GVN/PRE/pre-loop-load-through-select.ll +++ b/llvm/test/Transforms/GVN/PRE/pre-loop-load-through-select.ll @@ -19,8 +19,7 @@ ; CHECK-NEXT: [[EC:%.*]] = icmp eq i32* [[PTR_IV_NEXT]], [[END:%.*]] ; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: -; CHECK-NEXT: [[RES:%.*]] = load i32, i32* [[MIN_SELECT]], align 4 -; CHECK-NEXT: ret i32 [[RES]] +; CHECK-NEXT: ret i32 [[TMP0]] ; entry: %start.ptr = getelementptr inbounds i32, i32* %ptr, i64 1 @@ -60,8 +59,7 @@ ; CHECK-NEXT: [[EC:%.*]] = icmp eq i32* [[PTR_IV_NEXT]], [[END:%.*]] ; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: -; CHECK-NEXT: [[RES:%.*]] = load i32, i32* [[MIN_SELECT]], align 4 -; CHECK-NEXT: ret i32 [[RES]] +; CHECK-NEXT: ret i32 [[TMP0]] ; entry: %start.ptr = getelementptr inbounds i32, i32* %ptr, i64 1 @@ -101,8 +99,7 @@ ; CHECK-NEXT: [[EC:%.*]] = icmp eq i32* [[PTR_IV_NEXT]], [[END:%.*]] ; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: -; CHECK-NEXT: [[RES:%.*]] = load i32, i32* [[MIN_SELECT]], align 4 -; CHECK-NEXT: ret i32 [[RES]] +; CHECK-NEXT: ret i32 [[TMP0]] ; entry: br label %loop @@ -141,8 +138,7 @@ ; CHECK-NEXT: [[EC:%.*]] = icmp eq i32* [[PTR_IV_NEXT]], [[END:%.*]] ; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: -; CHECK-NEXT: [[RES:%.*]] = load i32, i32* [[MIN_SELECT]], align 4 -; CHECK-NEXT: ret i32 [[RES]] +; CHECK-NEXT: ret i32 [[TMP0]] ; entry: %start.ptr = getelementptr inbounds i32, i32* %ptr, i64 1 @@ -183,8 +179,7 @@ ; CHECK-NEXT: [[EC:%.*]] = icmp eq i32* [[PTR_IV_NEXT]], [[END:%.*]] ; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: -; CHECK-NEXT: [[RES:%.*]] = load i32, i32* [[MIN_SELECT]], align 4 -; CHECK-NEXT: ret i32 [[RES]] +; CHECK-NEXT: ret i32 [[TMP0]] ; entry: %start.ptr = getelementptr inbounds i32, i32* %ptr, i64 1 @@ -250,7 +245,10 @@ ; CHECK-LABEL: @test_pointer_phi_select_same_object_split_edge( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[START_PTR:%.*]] = getelementptr inbounds i32, i32* [[PTR:%.*]], i64 1 -; CHECK-NEXT: br i1 [[C:%.*]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]] +; CHECK-NEXT: br i1 [[C:%.*]], label [[ENTRY_EXIT_CRIT_EDGE:%.*]], label [[LOOP_PREHEADER:%.*]] +; CHECK: entry.exit_crit_edge: +; CHECK-NEXT: [[RES_PRE:%.*]] = load i32, i32* [[END:%.*]], align 4 +; CHECK-NEXT: br label [[EXIT:%.*]] ; CHECK: loop.preheader: ; CHECK-NEXT: [[L_2_PRE:%.*]] = load i32, i32* [[PTR]], align 4 ; CHECK-NEXT: br label [[LOOP:%.*]] @@ -263,13 +261,13 @@ ; CHECK-NEXT: [[TMP0]] = select i1 [[CMP_I_I_I]], i32 [[L_1]], i32 [[L_2]] ; CHECK-NEXT: [[MIN_SELECT]] = select i1 [[CMP_I_I_I]], i32* [[PTR_IV]], i32* [[MIN_PTR]] ; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i32, i32* [[PTR_IV]], i64 1 -; CHECK-NEXT: [[EC:%.*]] = icmp eq i32* [[PTR_IV_NEXT]], [[END:%.*]] +; CHECK-NEXT: [[EC:%.*]] = icmp eq i32* [[PTR_IV_NEXT]], [[END]] ; CHECK-NEXT: br i1 [[EC]], label [[LOOP_EXIT:%.*]], label [[LOOP]] ; CHECK: loop.exit: ; CHECK-NEXT: br label [[EXIT]] ; CHECK: exit: -; CHECK-NEXT: [[LCSSA_PHI_2:%.*]] = phi i32* [ [[END]], [[ENTRY:%.*]] ], [ [[MIN_SELECT]], [[LOOP_EXIT]] ] -; CHECK-NEXT: [[RES:%.*]] = load i32, i32* [[LCSSA_PHI_2]], align 4 +; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[RES_PRE]], [[ENTRY_EXIT_CRIT_EDGE]] ], [ [[TMP0]], [[LOOP_EXIT]] ] +; CHECK-NEXT: [[LCSSA_PHI_2:%.*]] = phi i32* [ [[END]], [[ENTRY_EXIT_CRIT_EDGE]] ], [ [[MIN_SELECT]], [[LOOP_EXIT]] ] ; CHECK-NEXT: ret i32 [[RES]] ; entry: @@ -311,13 +309,13 @@ ; CHECK-NEXT: [[L_1:%.*]] = load i32, i32* [[PTR_IV]], align 4 ; CHECK-NEXT: [[L_2:%.*]] = load i32, i32* [[MIN_PTR]], align 4 ; CHECK-NEXT: [[CMP_I_I_I:%.*]] = icmp ult i32 [[L_1]], [[L_2]] +; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[CMP_I_I_I]], i32 [[L_1]], i32 [[L_2]] ; CHECK-NEXT: [[MIN_SELECT]] = select i1 [[CMP_I_I_I]], i32* [[PTR_IV]], i32* [[MIN_PTR]] ; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i32, i32* [[PTR_IV]], i64 1 ; CHECK-NEXT: [[EC:%.*]] = icmp eq i32* [[PTR_IV_NEXT]], [[END:%.*]] ; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: -; CHECK-NEXT: [[RES:%.*]] = load i32, i32* [[MIN_SELECT]], align 4 -; CHECK-NEXT: ret i32 [[RES]] +; CHECK-NEXT: ret i32 [[TMP0]] ; entry: br label %loop @@ -350,13 +348,13 @@ ; CHECK-NEXT: call void @may_throw() ; CHECK-NEXT: [[L_2:%.*]] = load i32, i32* [[MIN_PTR]], align 4 ; CHECK-NEXT: [[CMP_I_I_I:%.*]] = icmp ult i32 [[L_1]], [[L_2]] +; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[CMP_I_I_I]], i32 [[L_1]], i32 [[L_2]] ; CHECK-NEXT: [[MIN_SELECT]] = select i1 [[CMP_I_I_I]], i32* [[PTR_IV]], i32* [[MIN_PTR]] ; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i32, i32* [[PTR_IV]], i64 1 ; CHECK-NEXT: [[EC:%.*]] = icmp eq i32* [[PTR_IV_NEXT]], [[END:%.*]] ; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: -; CHECK-NEXT: [[RES:%.*]] = load i32, i32* [[MIN_SELECT]], align 4 -; CHECK-NEXT: ret i32 [[RES]] +; CHECK-NEXT: ret i32 [[TMP0]] ; entry: br label %loop @@ -916,23 +914,18 @@ ; CHECK-NEXT: [[L_2_PRE:%.*]] = load i32, i32* [[PTR]], align 4 ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[L_3:%.*]] = phi i32 [ [[L_2_PRE]], [[ENTRY:%.*]] ], [ [[L_3_PRE:%.*]], [[LOOP_LOOP_CRIT_EDGE:%.*]] ] -; CHECK-NEXT: [[L_2:%.*]] = phi i32 [ [[L_2_PRE]], [[ENTRY]] ], [ [[TMP0:%.*]], [[LOOP_LOOP_CRIT_EDGE]] ] -; CHECK-NEXT: [[PTR_IV:%.*]] = phi i32* [ [[START_PTR]], [[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP_LOOP_CRIT_EDGE]] ] -; CHECK-NEXT: [[MIN_PTR:%.*]] = phi i32* [ [[PTR]], [[ENTRY]] ], [ [[MIN_SELECT:%.*]], [[LOOP_LOOP_CRIT_EDGE]] ] +; CHECK-NEXT: [[L_3:%.*]] = phi i32 [ [[L_2_PRE]], [[ENTRY:%.*]] ], [ [[TMP0:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[PTR_IV:%.*]] = phi i32* [ [[START_PTR]], [[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[MIN_PTR:%.*]] = phi i32* [ [[PTR]], [[ENTRY]] ], [ [[MIN_SELECT:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[L_1:%.*]] = load i32, i32* [[PTR_IV]], align 4 -; CHECK-NEXT: [[CMP_I_I_I:%.*]] = icmp ult i32 [[L_1]], [[L_2]] -; CHECK-NEXT: [[TMP0]] = select i1 [[CMP_I_I_I]], i32 [[L_1]], i32 [[L_2]] +; CHECK-NEXT: [[CMP_I_I_I:%.*]] = icmp ult i32 [[L_1]], [[L_3]] +; CHECK-NEXT: [[TMP0]] = select i1 [[CMP_I_I_I]], i32 [[L_1]], i32 [[L_3]] ; CHECK-NEXT: [[MIN_SELECT]] = select i1 [[CMP_I_I_I]], i32* [[PTR_IV]], i32* [[MIN_PTR]] ; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i32, i32* [[PTR_IV]], i32 [[L_3]] ; CHECK-NEXT: [[EC:%.*]] = icmp eq i32* [[PTR_IV_NEXT]], [[END:%.*]] -; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP_LOOP_CRIT_EDGE]] -; CHECK: loop.loop_crit_edge: -; CHECK-NEXT: [[L_3_PRE]] = load i32, i32* [[MIN_SELECT]], align 4 -; CHECK-NEXT: br label [[LOOP]] +; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: -; CHECK-NEXT: [[RES:%.*]] = load i32, i32* [[MIN_SELECT]], align 4 -; CHECK-NEXT: ret i32 [[RES]] +; CHECK-NEXT: ret i32 [[TMP0]] ; entry: %start.ptr = getelementptr inbounds i32, i32* %ptr, i64 1 @@ -975,8 +968,7 @@ ; CHECK-NEXT: [[EC:%.*]] = icmp eq i32* [[PTR_IV_NEXT]], [[END:%.*]] ; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: -; CHECK-NEXT: [[RES:%.*]] = load i32, i32* [[MIN_SELECT]], align 4 -; CHECK-NEXT: ret i32 [[RES]] +; CHECK-NEXT: ret i32 [[TMP0]] ; entry: %start.ptr = getelementptr inbounds i32, i32* %ptr, i64 1 diff --git a/llvm/test/Transforms/GVN/load-of-pointer-select-available.ll b/llvm/test/Transforms/GVN/load-of-pointer-select-available.ll --- a/llvm/test/Transforms/GVN/load-of-pointer-select-available.ll +++ b/llvm/test/Transforms/GVN/load-of-pointer-select-available.ll @@ -19,8 +19,7 @@ ; CHECK-NEXT: [[EC:%.*]] = icmp eq i32* [[PTR_IV_NEXT]], [[END:%.*]] ; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: -; CHECK-NEXT: [[RES:%.*]] = load i32, i32* [[MIN_SELECT]], align 4 -; CHECK-NEXT: ret i32 [[RES]] +; CHECK-NEXT: ret i32 [[MIN_VAL]] ; entry: %start.ptr = getelementptr inbounds i32, i32* %ptr, i64 1 @@ -872,8 +871,7 @@ ; CHECK-NEXT: [[EC:%.*]] = icmp eq i32* [[PTR_IV_NEXT]], [[END:%.*]] ; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: -; CHECK-NEXT: [[RES:%.*]] = load i32, i32* [[MIN_SELECT]], align 4 -; CHECK-NEXT: ret i32 [[RES]] +; CHECK-NEXT: ret i32 [[MIN_VAL]] ; entry: %start.ptr = getelementptr inbounds i32, i32* %ptr, i64 1 @@ -917,8 +915,7 @@ ; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: ; CHECK-NEXT: call void @may_throw() -; CHECK-NEXT: [[RES:%.*]] = load i32, i32* [[MIN_SELECT]], align 4 -; CHECK-NEXT: ret i32 [[RES]] +; CHECK-NEXT: ret i32 [[MIN_VAL]] ; entry: %start.ptr = getelementptr inbounds i32, i32* %ptr, i64 1