diff --git a/llvm/include/llvm/Transforms/Scalar/SROA.h b/llvm/include/llvm/Transforms/Scalar/SROA.h --- a/llvm/include/llvm/Transforms/Scalar/SROA.h +++ b/llvm/include/llvm/Transforms/Scalar/SROA.h @@ -26,6 +26,7 @@ class AllocaInst; class AssumptionCache; class DominatorTree; +class DomTreeUpdater; class Function; class LLVMContext; class PHINode; @@ -63,7 +64,7 @@ /// SSA vector values. class SROAPass : public PassInfoMixin { LLVMContext *C = nullptr; - DominatorTree *DT = nullptr; + DomTreeUpdater *DTU = nullptr; AssumptionCache *AC = nullptr; /// Worklist of alloca instructions to simplify. @@ -100,13 +101,9 @@ /// queue. SetVector> SpeculatablePHIs; - /// A worklist of select instructions to speculate prior to promoting + /// A worklist of select instructions to rewrite prior to promoting /// allocas. - /// - /// All of these select instructions have been checked for the safety of - /// speculation and by being speculated will allow promoting allocas - /// currently in the promotable queue. - SetVector> SpeculatableSelects; + SetVector> SelectsToRewrite; public: SROAPass() = default; @@ -119,6 +116,8 @@ friend class sroa::SROALegacyPass; /// Helper used by both the public run method and by the legacy pass. + PreservedAnalyses runImpl(Function &F, DomTreeUpdater &RunDTU, + AssumptionCache &RunAC); PreservedAnalyses runImpl(Function &F, DominatorTree &RunDT, AssumptionCache &RunAC); diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h --- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h +++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h @@ -464,10 +464,13 @@ /// ElseBlock /// SplitBefore /// Tail +/// +/// Updates DT if given. void SplitBlockAndInsertIfThenElse(Value *Cond, Instruction *SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, - MDNode *BranchWeights = nullptr); + MDNode *BranchWeights = nullptr, + DomTreeUpdater *DTU = nullptr); /// Check whether BB is the merge point of a if-region. /// If so, return the branch instruction that determines which entry into diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -38,6 +38,7 @@ #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/AssumptionCache.h" +#include "llvm/Analysis/DomTreeUpdater.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/PtrUseVisitor.h" @@ -78,6 +79,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Scalar.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/PromoteMemToReg.h" #include @@ -104,6 +106,7 @@ STATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced"); STATISTIC(NumPromoted, "Number of allocas promoted to SSA values"); STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion"); +STATISTIC(NumLoadsRewritten, "Number of loads rewritten to allow promotion"); STATISTIC(NumDeleted, "Number of instructions deleted"); STATISTIC(NumVectorized, "Number of vectorized aggregates"); @@ -1340,11 +1343,7 @@ /// We can do this to a select if its only uses are loads and if the operand /// to the select can be loaded unconditionally. If found an intervening bitcast /// with a single use of the load, allow the promotion. -static bool isSafeSelectToSpeculate(SelectInst &SI) { - Value *TValue = SI.getTrueValue(); - Value *FValue = SI.getFalseValue(); - const DataLayout &DL = SI.getModule()->getDataLayout(); - +static bool isSafeSelectToRewrite(SelectInst &SI) { for (User *U : SI.users()) { LoadInst *LI; BitCastInst *BC = dyn_cast(U); @@ -1353,29 +1352,42 @@ else LI = dyn_cast(U); - if (!LI || !LI->isSimple()) - return false; - - // Both operands to the select need to be dereferenceable, either - // absolutely (e.g. allocas) or at this point because we can see other - // accesses to it. - if (!isSafeToLoadUnconditionally(TValue, LI->getType(), - LI->getAlign(), DL, LI)) - return false; - if (!isSafeToLoadUnconditionally(FValue, LI->getType(), - LI->getAlign(), DL, LI)) + if (!LI) return false; } return true; } -static void speculateSelectInstLoads(IRBuilderTy &IRB, SelectInst &SI) { +static void rewriteLoadOfSelect(SelectInst &SI, LoadInst &LI, + DomTreeUpdater &DTU) { + BasicBlock *Head = LI.getParent(); + Instruction *ThenTerm = nullptr; + Instruction *ElseTerm = nullptr; + SplitBlockAndInsertIfThenElse(SI.getCondition(), &LI, &ThenTerm, &ElseTerm, + SI.getMetadata(LLVMContext::MD_prof), &DTU); + BasicBlock *Tail = LI.getParent(); + Tail->setName(Head->getName() + ".cont"); + auto *PN = PHINode::Create(LI.getType(), 2, "", &LI); + for (auto CondBlock : enumerate(successors(Head))) { + auto *BB = CondBlock.value(); + bool IsThen = CondBlock.index() == 0; + BB->setName(Head->getName() + (IsThen ? ".then" : ".else")); + auto *CondLoad = cast(LI.clone()); + CondLoad->insertBefore(BB->getTerminator()); + CondLoad->setOperand(0, SI.getOperand(1 + CondBlock.index())); + CondLoad->setName(LI.getName() + (IsThen ? ".then" : ".else") + ".val"); + PN->addIncoming(CondLoad, BB); + } + PN->takeName(&LI); + NumLoadsRewritten += 2; + LLVM_DEBUG(dbgs() << " to: " << *PN << "\n"); + LI.replaceAllUsesWith(PN); +} + +static void rewriteSelectInstLoads(SelectInst &SI, DomTreeUpdater &DTU) { LLVM_DEBUG(dbgs() << " original: " << SI << "\n"); - IRB.SetInsertPoint(&SI); - Value *TV = SI.getTrueValue(); - Value *FV = SI.getFalseValue(); // Replace the loads of the select with a select of two loads. while (!SI.use_empty()) { LoadInst *LI; @@ -1387,37 +1399,9 @@ LI = cast(SI.user_back()); } - assert(LI->isSimple() && "We only speculate simple loads"); - - IRB.SetInsertPoint(LI); - Value *NewTV = - BC ? IRB.CreateBitCast(TV, BC->getType(), TV->getName() + ".sroa.cast") - : TV; - Value *NewFV = - BC ? IRB.CreateBitCast(FV, BC->getType(), FV->getName() + ".sroa.cast") - : FV; - LoadInst *TL = IRB.CreateLoad(LI->getType(), NewTV, - LI->getName() + ".sroa.speculate.load.true"); - LoadInst *FL = IRB.CreateLoad(LI->getType(), NewFV, - LI->getName() + ".sroa.speculate.load.false"); - NumLoadsSpeculated += 2; - - // Transfer alignment and AA info if present. - TL->setAlignment(LI->getAlign()); - FL->setAlignment(LI->getAlign()); - - AAMDNodes Tags = LI->getAAMetadata(); - if (Tags) { - TL->setAAMetadata(Tags); - FL->setAAMetadata(Tags); - } - - Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL, - LI->getName() + ".sroa.speculated"); - - LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n"); - LI->replaceAllUsesWith(V); + rewriteLoadOfSelect(SI, *LI, DTU); LI->eraseFromParent(); + if (BC) BC->eraseFromParent(); } @@ -4367,7 +4351,7 @@ } for (SelectInst *Sel : SelectUsers) - if (!isSafeSelectToSpeculate(*Sel)) { + if (!isSafeSelectToRewrite(*Sel)) { Promotable = false; PHIUsers.clear(); SelectUsers.clear(); @@ -4392,7 +4376,7 @@ for (PHINode *PHIUser : PHIUsers) SpeculatablePHIs.insert(PHIUser); for (SelectInst *SelectUser : SelectUsers) - SpeculatableSelects.insert(SelectUser); + SelectsToRewrite.insert(SelectUser); Worklist.insert(NewAI); } } else { @@ -4665,9 +4649,9 @@ while (!SpeculatablePHIs.empty()) speculatePHINodeLoads(IRB, *SpeculatablePHIs.pop_back_val()); - LLVM_DEBUG(dbgs() << " Speculating Selects\n"); - while (!SpeculatableSelects.empty()) - speculateSelectInstLoads(IRB, *SpeculatableSelects.pop_back_val()); + LLVM_DEBUG(dbgs() << " Rewriting Selects\n"); + while (!SelectsToRewrite.empty()) + rewriteSelectInstLoads(*SelectsToRewrite.pop_back_val(), *DTU); return Changed; } @@ -4727,16 +4711,16 @@ NumPromoted += PromotableAllocas.size(); LLVM_DEBUG(dbgs() << "Promoting allocas with mem2reg...\n"); - PromoteMemToReg(PromotableAllocas, *DT, AC); + PromoteMemToReg(PromotableAllocas, DTU->getDomTree(), AC); PromotableAllocas.clear(); return true; } -PreservedAnalyses SROAPass::runImpl(Function &F, DominatorTree &RunDT, +PreservedAnalyses SROAPass::runImpl(Function &F, DomTreeUpdater &RunDTU, AssumptionCache &RunAC) { LLVM_DEBUG(dbgs() << "SROA function: " << F.getName() << "\n"); C = &F.getContext(); - DT = &RunDT; + DTU = &RunDTU; AC = &RunAC; BasicBlock &EntryBB = F.getEntryBlock(); @@ -4783,10 +4767,16 @@ return PreservedAnalyses::all(); PreservedAnalyses PA; - PA.preserveSet(); + PA.preserve(); return PA; } +PreservedAnalyses SROAPass::runImpl(Function &F, DominatorTree &RunDT, + AssumptionCache &RunAC) { + DomTreeUpdater DTU(RunDT, DomTreeUpdater::UpdateStrategy::Lazy); + return runImpl(F, DTU, RunAC); +} + PreservedAnalyses SROAPass::run(Function &F, FunctionAnalysisManager &AM) { return runImpl(F, AM.getResult(F), AM.getResult(F)); @@ -4821,7 +4811,7 @@ AU.addRequired(); AU.addRequired(); AU.addPreserved(); - AU.setPreservesCFG(); + AU.addPreserved(); } StringRef getPassName() const override { return "SROA"; } diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp --- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp +++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp @@ -1547,8 +1547,14 @@ void llvm::SplitBlockAndInsertIfThenElse(Value *Cond, Instruction *SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, - MDNode *BranchWeights) { + MDNode *BranchWeights, + DomTreeUpdater *DTU) { BasicBlock *Head = SplitBefore->getParent(); + + SmallPtrSet UniqueOrigSuccessors; + if (DTU) + UniqueOrigSuccessors.insert(succ_begin(Head), succ_end(Head)); + BasicBlock *Tail = Head->splitBasicBlock(SplitBefore->getIterator()); Instruction *HeadOldTerm = Head->getTerminator(); LLVMContext &C = Head->getContext(); @@ -1562,6 +1568,19 @@ BranchInst::Create(/*ifTrue*/ThenBlock, /*ifFalse*/ElseBlock, Cond); HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights); ReplaceInstWithInst(HeadOldTerm, HeadNewTerm); + if (DTU) { + SmallVector Updates; + Updates.reserve(4 + 2 * UniqueOrigSuccessors.size()); + for (BasicBlock *Succ : successors(Head)) { + Updates.push_back({DominatorTree::Insert, Head, Succ}); + Updates.push_back({DominatorTree::Insert, Succ, Tail}); + } + for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors) + Updates.push_back({DominatorTree::Insert, Tail, UniqueOrigSuccessor}); + for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors) + Updates.push_back({DominatorTree::Delete, Head, UniqueOrigSuccessor}); + DTU->applyUpdates(Updates); + } } BranchInst *llvm::GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue, diff --git a/llvm/test/Transforms/SROA/addrspacecast.ll b/llvm/test/Transforms/SROA/addrspacecast.ll --- a/llvm/test/Transforms/SROA/addrspacecast.ll +++ b/llvm/test/Transforms/SROA/addrspacecast.ll @@ -234,10 +234,14 @@ define void @select_addrspacecast_const_op(i1 %a, i1 %b) { ; CHECK-LABEL: @select_addrspacecast_const_op( -; CHECK-NEXT: [[C:%.*]] = alloca i64, align 8 -; CHECK-NEXT: [[C_0_ASC_SROA_CAST:%.*]] = addrspacecast ptr [[C]] to ptr addrspace(1) -; CHECK-NEXT: [[COND_IN:%.*]] = select i1 [[B:%.*]], ptr addrspace(1) [[C_0_ASC_SROA_CAST]], ptr addrspace(1) null -; CHECK-NEXT: [[COND:%.*]] = load i64, ptr addrspace(1) [[COND_IN]], align 8 +; CHECK-NEXT: br i1 [[B:%.*]], label [[DOTTHEN:%.*]], label [[DOTELSE:%.*]] +; CHECK: .then: +; CHECK-NEXT: br label [[DOTCONT:%.*]] +; CHECK: .else: +; CHECK-NEXT: [[COND_ELSE_VAL:%.*]] = load i64, ptr addrspace(1) null, align 8 +; CHECK-NEXT: br label [[DOTCONT]] +; CHECK: .cont: +; CHECK-NEXT: [[COND:%.*]] = phi i64 [ undef, [[DOTTHEN]] ], [ [[COND_ELSE_VAL]], [[DOTELSE]] ] ; CHECK-NEXT: ret void ; %c = alloca i64, align 8 @@ -255,8 +259,14 @@ define void @select_addrspacecast_gv(i1 %a, i1 %b) { ; CHECK-LABEL: @select_addrspacecast_gv( -; CHECK-NEXT: [[COND_SROA_SPECULATE_LOAD_FALSE:%.*]] = load i64, ptr addrspace(1) @gv, align 8 -; CHECK-NEXT: [[COND_SROA_SPECULATED:%.*]] = select i1 [[B:%.*]], i64 undef, i64 [[COND_SROA_SPECULATE_LOAD_FALSE]] +; CHECK-NEXT: br i1 [[B:%.*]], label [[DOTTHEN:%.*]], label [[DOTELSE:%.*]] +; CHECK: .then: +; CHECK-NEXT: br label [[DOTCONT:%.*]] +; CHECK: .else: +; CHECK-NEXT: [[COND_ELSE_VAL:%.*]] = load i64, ptr addrspace(1) @gv, align 8 +; CHECK-NEXT: br label [[DOTCONT]] +; CHECK: .cont: +; CHECK-NEXT: [[COND:%.*]] = phi i64 [ undef, [[DOTTHEN]] ], [ [[COND_ELSE_VAL]], [[DOTELSE]] ] ; CHECK-NEXT: ret void ; %c = alloca i64, align 8 @@ -270,8 +280,14 @@ define void @select_addrspacecast_gv_constexpr(i1 %a, i1 %b) { ; CHECK-LABEL: @select_addrspacecast_gv_constexpr( -; CHECK-NEXT: [[COND_SROA_SPECULATE_LOAD_FALSE:%.*]] = load i64, ptr addrspace(2) addrspacecast (ptr addrspace(1) @gv to ptr addrspace(2)), align 8 -; CHECK-NEXT: [[COND_SROA_SPECULATED:%.*]] = select i1 [[B:%.*]], i64 undef, i64 [[COND_SROA_SPECULATE_LOAD_FALSE]] +; CHECK-NEXT: br i1 [[B:%.*]], label [[DOTTHEN:%.*]], label [[DOTELSE:%.*]] +; CHECK: .then: +; CHECK-NEXT: br label [[DOTCONT:%.*]] +; CHECK: .else: +; CHECK-NEXT: [[COND_ELSE_VAL:%.*]] = load i64, ptr addrspace(2) addrspacecast (ptr addrspace(1) @gv to ptr addrspace(2)), align 8 +; CHECK-NEXT: br label [[DOTCONT]] +; CHECK: .cont: +; CHECK-NEXT: [[COND:%.*]] = phi i64 [ undef, [[DOTTHEN]] ], [ [[COND_ELSE_VAL]], [[DOTELSE]] ] ; CHECK-NEXT: ret void ; %c = alloca i64, align 8 @@ -285,8 +301,14 @@ define i8 @select_addrspacecast_i8(i1 %c) { ; CHECK-LABEL: @select_addrspacecast_i8( -; CHECK-NEXT: [[RET_SROA_SPECULATED:%.*]] = select i1 [[C:%.*]], i8 undef, i8 undef -; CHECK-NEXT: ret i8 [[RET_SROA_SPECULATED]] +; CHECK-NEXT: br i1 [[C:%.*]], label [[DOTTHEN:%.*]], label [[DOTELSE:%.*]] +; CHECK: .then: +; CHECK-NEXT: br label [[DOTCONT:%.*]] +; CHECK: .else: +; CHECK-NEXT: br label [[DOTCONT]] +; CHECK: .cont: +; CHECK-NEXT: [[RET:%.*]] = phi i8 [ undef, [[DOTTHEN]] ], [ undef, [[DOTELSE]] ] +; CHECK-NEXT: ret i8 [[RET]] ; %a = alloca i8 %b = alloca i8 diff --git a/llvm/test/Transforms/SROA/basictest.ll b/llvm/test/Transforms/SROA/basictest.ll --- a/llvm/test/Transforms/SROA/basictest.ll +++ b/llvm/test/Transforms/SROA/basictest.ll @@ -1423,7 +1423,13 @@ define void @PR15805(i1 %a, i1 %b) { ; CHECK-LABEL: @PR15805( -; CHECK-NEXT: [[COND_SROA_SPECULATED:%.*]] = select i1 [[B:%.*]], i64 undef, i64 undef +; CHECK-NEXT: br i1 [[B:%.*]], label [[DOTTHEN:%.*]], label [[DOTELSE:%.*]] +; CHECK: .then: +; CHECK-NEXT: br label [[DOTCONT:%.*]] +; CHECK: .else: +; CHECK-NEXT: br label [[DOTCONT]] +; CHECK: .cont: +; CHECK-NEXT: [[COND:%.*]] = phi i64 [ undef, [[DOTTHEN]] ], [ undef, [[DOTELSE]] ] ; CHECK-NEXT: ret void ; @@ -1442,7 +1448,15 @@ ; CHECK-LABEL: @PR15805.1( ; CHECK-NEXT: br label [[EXIT:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[COND_SROA_SPECULATED:%.*]] = select i1 [[A:%.*]], i64 undef, i64 undef +; CHECK-NEXT: br i1 [[A:%.*]], label [[LOOP_THEN:%.*]], label [[LOOP_ELSE:%.*]] +; CHECK: loop.then: +; CHECK-NEXT: [[C_0_LOAD:%.*]] = load i64, ptr poison, align 8 +; CHECK-NEXT: br label [[LOOP_CONT:%.*]] +; CHECK: loop.else: +; CHECK-NEXT: [[C_0_LOAD1:%.*]] = load i64, ptr poison, align 8 +; CHECK-NEXT: br label [[LOOP_CONT]] +; CHECK: loop.cont: +; CHECK-NEXT: [[COND:%.*]] = phi i64 [ [[C_0_LOAD]], [[LOOP_THEN]] ], [ [[C_0_LOAD1]], [[LOOP_ELSE]] ] ; CHECK-NEXT: br i1 [[C2:%.*]], label [[LOOP:%.*]], label [[EXIT]] ; CHECK: exit: ; CHECK-NEXT: ret void @@ -1499,10 +1513,15 @@ ; ; CHECK-LABEL: @PR16651.2( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TV1_SROA_0:%.*]] = alloca <2 x float>, align 8 -; CHECK-NEXT: store <2 x float> [[VAL:%.*]], ptr [[TV1_SROA_0]], align 8 -; CHECK-NEXT: [[COND105_IN_I_I:%.*]] = select i1 [[C1:%.*]], ptr null, ptr [[TV1_SROA_0]] -; CHECK-NEXT: [[COND105_I_I:%.*]] = load float, ptr [[COND105_IN_I_I]], align 8 +; CHECK-NEXT: br i1 [[C1:%.*]], label [[ENTRY_THEN:%.*]], label [[ENTRY_ELSE:%.*]] +; CHECK: entry.then: +; CHECK-NEXT: [[COND105_I_I_THEN_VAL:%.*]] = load float, ptr null, align 8 +; CHECK-NEXT: br label [[ENTRY_CONT:%.*]] +; CHECK: entry.else: +; CHECK-NEXT: [[TV1_SROA_0_0_VEC_EXTRACT:%.*]] = extractelement <2 x float> [[VAL:%.*]], i32 0 +; CHECK-NEXT: br label [[ENTRY_CONT]] +; CHECK: entry.cont: +; CHECK-NEXT: [[COND105_I_I:%.*]] = phi float [ [[COND105_I_I_THEN_VAL]], [[ENTRY_THEN]] ], [ [[TV1_SROA_0_0_VEC_EXTRACT]], [[ENTRY_ELSE]] ] ; CHECK-NEXT: ret void ; @@ -1774,9 +1793,9 @@ ; CHECK-LABEL: @PR25873( ; CHECK-NEXT: entry: ; CHECK-NEXT: store i32 1123418112, ptr [[OUTDATA:%.*]], align 4 -; CHECK-NEXT: [[DOTSROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[OUTDATA:%.*]], i64 4 -; CHECK-NEXT: store i32 1139015680, ptr [[DOTSROA_IDX]], align 4 -; CHECK-NEXT: [[TMPDATA_SROA_6_0__SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[OUTDATA:%.*]], i64 8 +; CHECK-NEXT: [[OUTDATA_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[OUTDATA]], i64 4 +; CHECK-NEXT: store i32 1139015680, ptr [[OUTDATA_SROA_IDX]], align 4 +; CHECK-NEXT: [[TMPDATA_SROA_6_0_OUTDATA_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[OUTDATA]], i64 8 ; CHECK-NEXT: [[TMPDATA_SROA_6_SROA_4_0_INSERT_EXT:%.*]] = zext i32 1139015680 to i64 ; CHECK-NEXT: [[TMPDATA_SROA_6_SROA_4_0_INSERT_SHIFT:%.*]] = shl i64 [[TMPDATA_SROA_6_SROA_4_0_INSERT_EXT]], 32 ; CHECK-NEXT: [[TMPDATA_SROA_6_SROA_4_0_INSERT_MASK:%.*]] = and i64 undef, 4294967295 @@ -1784,7 +1803,7 @@ ; CHECK-NEXT: [[TMPDATA_SROA_6_SROA_0_0_INSERT_EXT:%.*]] = zext i32 1123418112 to i64 ; CHECK-NEXT: [[TMPDATA_SROA_6_SROA_0_0_INSERT_MASK:%.*]] = and i64 [[TMPDATA_SROA_6_SROA_4_0_INSERT_INSERT]], -4294967296 ; CHECK-NEXT: [[TMPDATA_SROA_6_SROA_0_0_INSERT_INSERT:%.*]] = or i64 [[TMPDATA_SROA_6_SROA_0_0_INSERT_MASK]], [[TMPDATA_SROA_6_SROA_0_0_INSERT_EXT]] -; CHECK-NEXT: store i64 [[TMPDATA_SROA_6_SROA_0_0_INSERT_INSERT]], ptr [[TMPDATA_SROA_6_0__SROA_IDX]], align 4 +; CHECK-NEXT: store i64 [[TMPDATA_SROA_6_SROA_0_0_INSERT_INSERT]], ptr [[TMPDATA_SROA_6_0_OUTDATA_SROA_IDX]], align 4 ; CHECK-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/SROA/phi-and-select.ll b/llvm/test/Transforms/SROA/phi-and-select.ll --- a/llvm/test/Transforms/SROA/phi-and-select.ll +++ b/llvm/test/Transforms/SROA/phi-and-select.ll @@ -39,8 +39,14 @@ ; CHECK-LABEL: @test2( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[COND:%.*]] = icmp sle i32 0, 1 -; CHECK-NEXT: [[RESULT_SROA_SPECULATED:%.*]] = select i1 [[COND]], i32 1, i32 0 -; CHECK-NEXT: ret i32 [[RESULT_SROA_SPECULATED]] +; CHECK-NEXT: br i1 [[COND]], label [[ENTRY_THEN:%.*]], label [[ENTRY_ELSE:%.*]] +; CHECK: entry.then: +; CHECK-NEXT: br label [[ENTRY_CONT:%.*]] +; CHECK: entry.else: +; CHECK-NEXT: br label [[ENTRY_CONT]] +; CHECK: entry.cont: +; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ 1, [[ENTRY_THEN]] ], [ 0, [[ENTRY_ELSE]] ] +; CHECK-NEXT: ret i32 [[RESULT]] ; entry: %a = alloca [2 x i32] @@ -62,10 +68,16 @@ ; CHECK-LABEL: @test2_bitcast( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[COND:%.*]] = icmp sle i32 0, 1 +; CHECK-NEXT: br i1 [[COND]], label [[ENTRY_THEN:%.*]], label [[ENTRY_ELSE:%.*]] +; CHECK: entry.then: ; CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 1 to float +; CHECK-NEXT: br label [[ENTRY_CONT:%.*]] +; CHECK: entry.else: ; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 0 to float -; CHECK-NEXT: [[RESULT_SROA_SPECULATED:%.*]] = select i1 [[COND]], float [[TMP0]], float [[TMP1]] -; CHECK-NEXT: ret float [[RESULT_SROA_SPECULATED]] +; CHECK-NEXT: br label [[ENTRY_CONT]] +; CHECK: entry.cont: +; CHECK-NEXT: [[RESULT:%.*]] = phi float [ [[TMP0]], [[ENTRY_THEN]] ], [ [[TMP1]], [[ENTRY_ELSE]] ] +; CHECK-NEXT: ret float [[RESULT]] ; entry: %a = alloca [2 x i32] @@ -330,9 +342,15 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: store i32 0, ptr [[PTR:%.*]], align 4 ; CHECK-NEXT: [[TEST:%.*]] = icmp ne i32 [[B:%.*]], 0 -; CHECK-NEXT: [[LOADED_SROA_SPECULATE_LOAD_FALSE:%.*]] = load i32, ptr [[PTR]], align 4 -; CHECK-NEXT: [[LOADED_SROA_SPECULATED:%.*]] = select i1 [[TEST]], i32 undef, i32 [[LOADED_SROA_SPECULATE_LOAD_FALSE]] -; CHECK-NEXT: ret i32 [[LOADED_SROA_SPECULATED]] +; CHECK-NEXT: br i1 [[TEST]], label [[ENTRY_THEN:%.*]], label [[ENTRY_ELSE:%.*]] +; CHECK: entry.then: +; CHECK-NEXT: br label [[ENTRY_CONT:%.*]] +; CHECK: entry.else: +; CHECK-NEXT: [[LOADED_ELSE_VAL:%.*]] = load i32, ptr [[PTR]], align 4 +; CHECK-NEXT: br label [[ENTRY_CONT]] +; CHECK: entry.cont: +; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ undef, [[ENTRY_THEN]] ], [ [[LOADED_ELSE_VAL]], [[ENTRY_ELSE]] ] +; CHECK-NEXT: ret i32 [[LOADED]] ; entry: @@ -391,10 +409,16 @@ ; CHECK-NEXT: store double 0.000000e+00, ptr [[F]], align 8 ; CHECK-NEXT: store float 0.000000e+00, ptr [[PTR:%.*]], align 4 ; CHECK-NEXT: [[TEST:%.*]] = icmp ne i32 [[B:%.*]], 0 -; CHECK-NEXT: [[F_0_LOADED_SROA_SPECULATE_LOAD_TRUE:%.*]] = load float, ptr [[F]], align 8 -; CHECK-NEXT: [[LOADED_SROA_SPECULATE_LOAD_FALSE:%.*]] = load float, ptr [[PTR]], align 4 -; CHECK-NEXT: [[LOADED_SROA_SPECULATED:%.*]] = select i1 [[TEST]], float [[F_0_LOADED_SROA_SPECULATE_LOAD_TRUE]], float [[LOADED_SROA_SPECULATE_LOAD_FALSE]] -; CHECK-NEXT: ret float [[LOADED_SROA_SPECULATED]] +; CHECK-NEXT: br i1 [[TEST]], label [[ENTRY_THEN:%.*]], label [[ENTRY_ELSE:%.*]] +; CHECK: entry.then: +; CHECK-NEXT: [[F_0_LOADED_THEN_VAL:%.*]] = load float, ptr [[F]], align 8 +; CHECK-NEXT: br label [[ENTRY_CONT:%.*]] +; CHECK: entry.else: +; CHECK-NEXT: [[LOADED_ELSE_VAL:%.*]] = load float, ptr [[PTR]], align 4 +; CHECK-NEXT: br label [[ENTRY_CONT]] +; CHECK: entry.cont: +; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[F_0_LOADED_THEN_VAL]], [[ENTRY_THEN]] ], [ [[LOADED_ELSE_VAL]], [[ENTRY_ELSE]] ] +; CHECK-NEXT: ret float [[LOADED]] ; entry: @@ -813,11 +837,15 @@ ; Don't speculate a load based on an earlier volatile operation. define i8 @volatile_select(ptr %p, i1 %b) { ; CHECK-LABEL: @volatile_select( -; CHECK-NEXT: [[P2:%.*]] = alloca i8, align 1 -; CHECK-NEXT: store i8 0, ptr [[P2]], align 1 ; CHECK-NEXT: store volatile i8 0, ptr [[P:%.*]], align 1 -; CHECK-NEXT: [[PX:%.*]] = select i1 [[B:%.*]], ptr [[P]], ptr [[P2]] -; CHECK-NEXT: [[V2:%.*]] = load i8, ptr [[PX]], align 1 +; CHECK-NEXT: br i1 [[B:%.*]], label [[DOTTHEN:%.*]], label [[DOTELSE:%.*]] +; CHECK: .then: +; CHECK-NEXT: [[V2_THEN_VAL:%.*]] = load i8, ptr [[P]], align 1 +; CHECK-NEXT: br label [[DOTCONT:%.*]] +; CHECK: .else: +; CHECK-NEXT: br label [[DOTCONT]] +; CHECK: .cont: +; CHECK-NEXT: [[V2:%.*]] = phi i8 [ [[V2_THEN_VAL]], [[DOTTHEN]] ], [ 0, [[DOTELSE]] ] ; CHECK-NEXT: ret i8 [[V2]] ; %p2 = alloca i8 diff --git a/llvm/test/Transforms/SROA/select-gep.ll b/llvm/test/Transforms/SROA/select-gep.ll --- a/llvm/test/Transforms/SROA/select-gep.ll +++ b/llvm/test/Transforms/SROA/select-gep.ll @@ -6,8 +6,14 @@ define i32 @test_sroa_select_gep(i1 %cond) { ; CHECK-LABEL: @test_sroa_select_gep( ; CHECK-NEXT: bb: -; CHECK-NEXT: [[LOAD_SROA_SPECULATED:%.*]] = select i1 [[COND:%.*]], i32 1, i32 2 -; CHECK-NEXT: ret i32 [[LOAD_SROA_SPECULATED]] +; CHECK-NEXT: br i1 [[COND:%.*]], label [[BB_THEN:%.*]], label [[BB_ELSE:%.*]] +; CHECK: bb.then: +; CHECK-NEXT: br label [[BB_CONT:%.*]] +; CHECK: bb.else: +; CHECK-NEXT: br label [[BB_CONT]] +; CHECK: bb.cont: +; CHECK-NEXT: [[LOAD:%.*]] = phi i32 [ 1, [[BB_THEN]] ], [ 2, [[BB_ELSE]] ] +; CHECK-NEXT: ret i32 [[LOAD]] ; bb: %a = alloca %pair, align 4 @@ -25,8 +31,14 @@ define i32 @test_sroa_select_gep_non_inbound(i1 %cond) { ; CHECK-LABEL: @test_sroa_select_gep_non_inbound( ; CHECK-NEXT: bb: -; CHECK-NEXT: [[LOAD_SROA_SPECULATED:%.*]] = select i1 [[COND:%.*]], i32 1, i32 2 -; CHECK-NEXT: ret i32 [[LOAD_SROA_SPECULATED]] +; CHECK-NEXT: br i1 [[COND:%.*]], label [[BB_THEN:%.*]], label [[BB_ELSE:%.*]] +; CHECK: bb.then: +; CHECK-NEXT: br label [[BB_CONT:%.*]] +; CHECK: bb.else: +; CHECK-NEXT: br label [[BB_CONT]] +; CHECK: bb.cont: +; CHECK-NEXT: [[LOAD:%.*]] = phi i32 [ 1, [[BB_THEN]] ], [ 2, [[BB_ELSE]] ] +; CHECK-NEXT: ret i32 [[LOAD]] ; bb: %a = alloca %pair, align 4 @@ -52,10 +64,24 @@ ; CHECK-NEXT: store i32 12, ptr [[B_SROA_0]], align 4 ; CHECK-NEXT: store i32 21, ptr [[A_SROA_2]], align 4 ; CHECK-NEXT: store i32 22, ptr [[B_SROA_2]], align 4 -; CHECK-NEXT: [[SELECT_SROA_SEL:%.*]] = select i1 [[COND:%.*]], ptr [[A_SROA_0]], ptr [[B_SROA_0]] -; CHECK-NEXT: [[LOAD1:%.*]] = load volatile i32, ptr [[SELECT_SROA_SEL]], align 4 -; CHECK-NEXT: [[SELECT_SROA_SEL3:%.*]] = select i1 [[COND]], ptr [[A_SROA_2]], ptr [[B_SROA_2]] -; CHECK-NEXT: [[LOAD2:%.*]] = load volatile i32, ptr [[SELECT_SROA_SEL3]], align 4 +; CHECK-NEXT: br i1 [[COND:%.*]], label [[BB_THEN2:%.*]], label [[BB_ELSE3:%.*]] +; CHECK: bb.then2: +; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_LOAD1_THEN_VAL:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 4 +; CHECK-NEXT: br label [[BB_CONT1:%.*]] +; CHECK: bb.else3: +; CHECK-NEXT: [[B_SROA_0_0_LOAD1_ELSE_VAL:%.*]] = load volatile i32, ptr [[B_SROA_0]], align 4 +; CHECK-NEXT: br label [[BB_CONT1]] +; CHECK: bb.cont1: +; CHECK-NEXT: [[LOAD1:%.*]] = phi i32 [ [[A_SROA_0_0_A_SROA_0_0_LOAD1_THEN_VAL]], [[BB_THEN2]] ], [ [[B_SROA_0_0_LOAD1_ELSE_VAL]], [[BB_ELSE3]] ] +; CHECK-NEXT: br i1 [[COND]], label [[BB_THEN:%.*]], label [[BB_ELSE:%.*]] +; CHECK: bb.then: +; CHECK-NEXT: [[A_SROA_2_0_A_SROA_2_4_LOAD2_THEN_VAL:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4 +; CHECK-NEXT: br label [[BB_CONT:%.*]] +; CHECK: bb.else: +; CHECK-NEXT: [[B_SROA_2_0_LOAD2_ELSE_VAL:%.*]] = load volatile i32, ptr [[B_SROA_2]], align 4 +; CHECK-NEXT: br label [[BB_CONT]] +; CHECK: bb.cont: +; CHECK-NEXT: [[LOAD2:%.*]] = phi i32 [ [[A_SROA_2_0_A_SROA_2_4_LOAD2_THEN_VAL]], [[BB_THEN]] ], [ [[B_SROA_2_0_LOAD2_ELSE_VAL]], [[BB_ELSE]] ] ; CHECK-NEXT: [[ADD:%.*]] = add i32 [[LOAD1]], [[LOAD2]] ; CHECK-NEXT: ret i32 [[ADD]] ; @@ -79,9 +105,14 @@ define i32 @test_sroa_select_gep_poison(i1 %cond) { ; CHECK-LABEL: @test_sroa_select_gep_poison( ; CHECK-NEXT: bb: -; CHECK-NEXT: [[A_SROA_0:%.*]] = alloca i32, align 4 -; CHECK-NEXT: [[SELECT_SROA_SEL:%.*]] = select i1 [[COND:%.*]], ptr [[A_SROA_0]], ptr poison -; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[SELECT_SROA_SEL]], align 4 +; CHECK-NEXT: br i1 [[COND:%.*]], label [[BB_THEN:%.*]], label [[BB_ELSE:%.*]] +; CHECK: bb.then: +; CHECK-NEXT: br label [[BB_CONT:%.*]] +; CHECK: bb.else: +; CHECK-NEXT: [[LOAD_ELSE_VAL:%.*]] = load i32, ptr poison, align 4 +; CHECK-NEXT: br label [[BB_CONT]] +; CHECK: bb.cont: +; CHECK-NEXT: [[LOAD:%.*]] = phi i32 [ undef, [[BB_THEN]] ], [ [[LOAD_ELSE_VAL]], [[BB_ELSE]] ] ; CHECK-NEXT: ret i32 [[LOAD]] ; bb: @@ -99,8 +130,8 @@ ; CHECK-NEXT: [[B_SROA_0:%.*]] = alloca i32, align 4 ; CHECK-NEXT: store i32 1, ptr [[A_SROA_0]], align 4 ; CHECK-NEXT: store i32 2, ptr [[B_SROA_0]], align 4 -; CHECK-NEXT: [[SELECT_SROA_SEL:%.*]] = select i1 [[COND:%.*]], ptr [[A_SROA_0]], ptr [[B_SROA_0]] -; CHECK-NEXT: [[SELECT2:%.*]] = select i1 [[COND]], ptr [[SELECT_SROA_SEL]], ptr [[A_SROA_0]] +; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[COND:%.*]], ptr [[A_SROA_0]], ptr [[B_SROA_0]] +; CHECK-NEXT: [[SELECT2:%.*]] = select i1 [[COND]], ptr [[SELECT]], ptr [[A_SROA_0]] ; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[SELECT2]], align 4 ; CHECK-NEXT: ret i32 [[LOAD]] ; diff --git a/llvm/test/Transforms/SROA/select-load.ll b/llvm/test/Transforms/SROA/select-load.ll --- a/llvm/test/Transforms/SROA/select-load.ll +++ b/llvm/test/Transforms/SROA/select-load.ll @@ -8,14 +8,26 @@ define <2 x i16> @test_load_bitcast_select(i1 %cond1, i1 %cond2) { ; CHECK-LABEL: @test_load_bitcast_select( ; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND1:%.*]], label [[ENTRY_THEN2:%.*]], label [[ENTRY_ELSE3:%.*]] +; CHECK: entry.then2: ; CHECK-NEXT: [[TMP0:%.*]] = bitcast half 0xHFFFF to i16 +; CHECK-NEXT: br label [[ENTRY_CONT1:%.*]] +; CHECK: entry.else3: ; CHECK-NEXT: [[TMP1:%.*]] = bitcast half 0xH0000 to i16 -; CHECK-NEXT: [[LD1_SROA_SPECULATED:%.*]] = select i1 [[COND1:%.*]], i16 [[TMP0]], i16 [[TMP1]] -; CHECK-NEXT: [[V1:%.*]] = insertelement <2 x i16> poison, i16 [[LD1_SROA_SPECULATED]], i32 0 +; CHECK-NEXT: br label [[ENTRY_CONT1]] +; CHECK: entry.cont1: +; CHECK-NEXT: [[LD1:%.*]] = phi i16 [ [[TMP0]], [[ENTRY_THEN2]] ], [ [[TMP1]], [[ENTRY_ELSE3]] ] +; CHECK-NEXT: [[V1:%.*]] = insertelement <2 x i16> poison, i16 [[LD1]], i32 0 +; CHECK-NEXT: br i1 [[COND2:%.*]], label [[ENTRY_THEN:%.*]], label [[ENTRY_ELSE:%.*]] +; CHECK: entry.then: ; CHECK-NEXT: [[TMP2:%.*]] = bitcast half 0xHFFFF to i16 +; CHECK-NEXT: br label [[ENTRY_CONT:%.*]] +; CHECK: entry.else: ; CHECK-NEXT: [[TMP3:%.*]] = bitcast half 0xH0000 to i16 -; CHECK-NEXT: [[LD2_SROA_SPECULATED:%.*]] = select i1 [[COND2:%.*]], i16 [[TMP2]], i16 [[TMP3]] -; CHECK-NEXT: [[V2:%.*]] = insertelement <2 x i16> [[V1]], i16 [[LD2_SROA_SPECULATED]], i32 1 +; CHECK-NEXT: br label [[ENTRY_CONT]] +; CHECK: entry.cont: +; CHECK-NEXT: [[LD2:%.*]] = phi i16 [ [[TMP2]], [[ENTRY_THEN]] ], [ [[TMP3]], [[ENTRY_ELSE]] ] +; CHECK-NEXT: [[V2:%.*]] = insertelement <2 x i16> [[V1]], i16 [[LD2]], i32 1 ; CHECK-NEXT: ret <2 x i16> [[V2]] ; entry: @@ -38,10 +50,22 @@ define void @test_multiple_loads_select(i1 %cmp){ ; CHECK-LABEL: @test_multiple_loads_select( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[ADDR_I8_SROA_SPECULATED:%.*]] = select i1 [[CMP:%.*]], ptr undef, ptr undef -; CHECK-NEXT: call void @foo_i8(ptr [[ADDR_I8_SROA_SPECULATED]]) -; CHECK-NEXT: [[ADDR_I32_SROA_SPECULATED:%.*]] = select i1 [[CMP]], ptr undef, ptr undef -; CHECK-NEXT: call void @foo_i32(ptr [[ADDR_I32_SROA_SPECULATED]]) +; CHECK-NEXT: br i1 [[CMP:%.*]], label [[ENTRY_THEN:%.*]], label [[ENTRY_ELSE:%.*]] +; CHECK: entry.then: +; CHECK-NEXT: br label [[ENTRY_CONT:%.*]] +; CHECK: entry.else: +; CHECK-NEXT: br label [[ENTRY_CONT]] +; CHECK: entry.cont: +; CHECK-NEXT: [[ADDR_I8:%.*]] = phi ptr [ undef, [[ENTRY_THEN]] ], [ undef, [[ENTRY_ELSE]] ] +; CHECK-NEXT: call void @foo_i8(ptr [[ADDR_I8]]) +; CHECK-NEXT: br i1 [[CMP]], label [[ENTRY_CONT_THEN:%.*]], label [[ENTRY_CONT_ELSE:%.*]] +; CHECK: entry.cont.then: +; CHECK-NEXT: br label [[ENTRY_CONT_CONT:%.*]] +; CHECK: entry.cont.else: +; CHECK-NEXT: br label [[ENTRY_CONT_CONT]] +; CHECK: entry.cont.cont: +; CHECK-NEXT: [[ADDR_I32:%.*]] = phi ptr [ undef, [[ENTRY_CONT_THEN]] ], [ undef, [[ENTRY_CONT_ELSE]] ] +; CHECK-NEXT: call void @foo_i32(ptr [[ADDR_I32]]) ; CHECK-NEXT: ret void ; entry: @@ -65,9 +89,15 @@ ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], i64 [[INDVARS_IV:%.*]] ; CHECK-NEXT: [[I1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 ; CHECK-NEXT: [[CMP_I_I:%.*]] = icmp slt i32 [[I1]], 0 -; CHECK-NEXT: [[I3_SROA_SPECULATE_LOAD_FALSE:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 -; CHECK-NEXT: [[I3_SROA_SPECULATED:%.*]] = select i1 [[CMP_I_I]], i32 0, i32 [[I3_SROA_SPECULATE_LOAD_FALSE]] -; CHECK-NEXT: ret i32 [[I3_SROA_SPECULATED]] +; CHECK-NEXT: br i1 [[CMP_I_I]], label [[DOTTHEN:%.*]], label [[DOTELSE:%.*]] +; CHECK: .then: +; CHECK-NEXT: br label [[DOTCONT:%.*]] +; CHECK: .else: +; CHECK-NEXT: [[I3_ELSE_VAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: br label [[DOTCONT]] +; CHECK: .cont: +; CHECK-NEXT: [[I3:%.*]] = phi i32 [ 0, [[DOTTHEN]] ], [ [[I3_ELSE_VAL]], [[DOTELSE]] ] +; CHECK-NEXT: ret i32 [[I3]] ; %min = alloca i32, align 4 %arrayidx = getelementptr inbounds i32, ptr %data, i64 %indvars.iv @@ -83,20 +113,26 @@ ; We should recursively evaluate select's. define i32 @clamp_load_to_constant_range(ptr %data, i64 %indvars.iv) { ; CHECK-LABEL: @clamp_load_to_constant_range( -; CHECK-NEXT: [[MIN:%.*]] = alloca i32, align 4 -; CHECK-NEXT: [[MAX:%.*]] = alloca i32, align 4 ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], i64 [[INDVARS_IV:%.*]] -; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[MIN]]) -; CHECK-NEXT: store i32 0, ptr [[MIN]], align 4 -; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[MAX]]) -; CHECK-NEXT: store i32 4095, ptr [[MAX]], align 4 ; CHECK-NEXT: [[I1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 ; CHECK-NEXT: [[CMP_I_I:%.*]] = icmp slt i32 [[I1]], 0 ; CHECK-NEXT: [[I2:%.*]] = tail call i32 @llvm.smax.i32(i32 [[I1]], i32 0) -; CHECK-NEXT: [[__B___A_I_I:%.*]] = select i1 [[CMP_I_I]], ptr [[MIN]], ptr [[ARRAYIDX]] ; CHECK-NEXT: [[CMP_I1_I:%.*]] = icmp ugt i32 [[I2]], 4095 -; CHECK-NEXT: [[__B___A_I2_I:%.*]] = select i1 [[CMP_I1_I]], ptr [[MAX]], ptr [[__B___A_I_I]] -; CHECK-NEXT: [[I3:%.*]] = load i32, ptr [[__B___A_I2_I]], align 4 +; CHECK-NEXT: br i1 [[CMP_I1_I]], label [[DOTTHEN:%.*]], label [[DOTELSE:%.*]] +; CHECK: .then: +; CHECK-NEXT: br label [[DOTCONT:%.*]] +; CHECK: .else: +; CHECK-NEXT: br i1 [[CMP_I_I]], label [[DOTELSE_THEN:%.*]], label [[DOTELSE_ELSE:%.*]] +; CHECK: .else.then: +; CHECK-NEXT: br label [[DOTELSE_CONT:%.*]] +; CHECK: .else.else: +; CHECK-NEXT: [[I3_ELSE_VAL_ELSE_VAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: br label [[DOTELSE_CONT]] +; CHECK: .else.cont: +; CHECK-NEXT: [[I3_ELSE_VAL:%.*]] = phi i32 [ 0, [[DOTELSE_THEN]] ], [ [[I3_ELSE_VAL_ELSE_VAL]], [[DOTELSE_ELSE]] ] +; CHECK-NEXT: br label [[DOTCONT]] +; CHECK: .cont: +; CHECK-NEXT: [[I3:%.*]] = phi i32 [ 4095, [[DOTTHEN]] ], [ [[I3_ELSE_VAL]], [[DOTELSE_CONT]] ] ; CHECK-NEXT: ret i32 [[I3]] ; %min = alloca i32, align 4 @@ -119,10 +155,14 @@ define i32 @non_speculatable_load_of_select(i1 %cond, ptr %data) { ; CHECK-LABEL: @non_speculatable_load_of_select( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[MIN:%.*]] = alloca i32, align 4 -; CHECK-NEXT: store i32 0, ptr [[MIN]], align 4 -; CHECK-NEXT: [[ADDR:%.*]] = select i1 [[COND:%.*]], ptr [[MIN]], ptr [[DATA:%.*]], !prof [[PROF0:![0-9]+]] -; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[ADDR]], align 4 +; CHECK-NEXT: br i1 [[COND:%.*]], label [[ENTRY_THEN:%.*]], label [[ENTRY_ELSE:%.*]], !prof [[PROF0:![0-9]+]] +; CHECK: entry.then: +; CHECK-NEXT: br label [[ENTRY_CONT:%.*]] +; CHECK: entry.else: +; CHECK-NEXT: [[R_ELSE_VAL:%.*]] = load i32, ptr [[DATA:%.*]], align 4 +; CHECK-NEXT: br label [[ENTRY_CONT]] +; CHECK: entry.cont: +; CHECK-NEXT: [[R:%.*]] = phi i32 [ 0, [[ENTRY_THEN]] ], [ [[R_ELSE_VAL]], [[ENTRY_ELSE]] ] ; CHECK-NEXT: ret i32 [[R]] ; entry: @@ -135,10 +175,14 @@ define i32 @non_speculatable_load_of_select_inverted(i1 %cond, ptr %data) { ; CHECK-LABEL: @non_speculatable_load_of_select_inverted( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[MAX:%.*]] = alloca i32, align 4 -; CHECK-NEXT: store i32 4095, ptr [[MAX]], align 4 -; CHECK-NEXT: [[ADDR:%.*]] = select i1 [[COND:%.*]], ptr [[DATA:%.*]], ptr [[MAX]], !prof [[PROF0]] -; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[ADDR]], align 4 +; CHECK-NEXT: br i1 [[COND:%.*]], label [[ENTRY_THEN:%.*]], label [[ENTRY_ELSE:%.*]], !prof [[PROF0]] +; CHECK: entry.then: +; CHECK-NEXT: [[R_THEN_VAL:%.*]] = load i32, ptr [[DATA:%.*]], align 4 +; CHECK-NEXT: br label [[ENTRY_CONT:%.*]] +; CHECK: entry.else: +; CHECK-NEXT: br label [[ENTRY_CONT]] +; CHECK: entry.cont: +; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[R_THEN_VAL]], [[ENTRY_THEN]] ], [ 4095, [[ENTRY_ELSE]] ] ; CHECK-NEXT: ret i32 [[R]] ; entry: @@ -152,7 +196,7 @@ !0 = !{!"branch_weights", i32 1, i32 99} ; Ensure that the branch metadata is reversed to match the reversals above. -; CHECK: !0 = {{.*}} i32 1, i32 99} +; CHECK: [[PROF0]] = {{.*}} i32 1, i32 99} declare void @llvm.lifetime.start.p0(i64, ptr ) declare void @llvm.lifetime.end.p0(i64, ptr) diff --git a/llvm/test/Transforms/SROA/std-clamp.ll b/llvm/test/Transforms/SROA/std-clamp.ll --- a/llvm/test/Transforms/SROA/std-clamp.ll +++ b/llvm/test/Transforms/SROA/std-clamp.ll @@ -10,10 +10,22 @@ ; CHECK-NEXT: store float [[HI:%.*]], ptr [[I4]], align 4 ; CHECK-NEXT: [[I5:%.*]] = fcmp fast olt float [[X:%.*]], [[LO:%.*]] ; CHECK-NEXT: [[I6:%.*]] = fcmp fast olt float [[HI]], [[X]] -; CHECK-NEXT: [[I9_SROA_SPECULATE_LOAD_FALSE_SROA_SPECULATE_LOAD_TRUE:%.*]] = load float, ptr [[I4]], align 4 -; CHECK-NEXT: [[I9_SROA_SPECULATE_LOAD_FALSE_SROA_SPECULATED:%.*]] = select i1 [[I6]], float [[I9_SROA_SPECULATE_LOAD_FALSE_SROA_SPECULATE_LOAD_TRUE]], float [[X]] -; CHECK-NEXT: [[I9_SROA_SPECULATED:%.*]] = select i1 [[I5]], float [[LO]], float [[I9_SROA_SPECULATE_LOAD_FALSE_SROA_SPECULATED]] -; CHECK-NEXT: ret float [[I9_SROA_SPECULATED]] +; CHECK-NEXT: br i1 [[I5]], label [[BB_THEN:%.*]], label [[BB_ELSE:%.*]] +; CHECK: bb.then: +; CHECK-NEXT: br label [[BB_CONT:%.*]] +; CHECK: bb.else: +; CHECK-NEXT: br i1 [[I6]], label [[BB_ELSE_THEN:%.*]], label [[BB_ELSE_ELSE:%.*]] +; CHECK: bb.else.then: +; CHECK-NEXT: [[I9_ELSE_VAL_THEN_VAL:%.*]] = load float, ptr [[I4]], align 4 +; CHECK-NEXT: br label [[BB_ELSE_CONT:%.*]] +; CHECK: bb.else.else: +; CHECK-NEXT: br label [[BB_ELSE_CONT]] +; CHECK: bb.else.cont: +; CHECK-NEXT: [[I9_ELSE_VAL:%.*]] = phi float [ [[I9_ELSE_VAL_THEN_VAL]], [[BB_ELSE_THEN]] ], [ [[X]], [[BB_ELSE_ELSE]] ] +; CHECK-NEXT: br label [[BB_CONT]] +; CHECK: bb.cont: +; CHECK-NEXT: [[I9:%.*]] = phi float [ [[LO]], [[BB_THEN]] ], [ [[I9_ELSE_VAL]], [[BB_ELSE_CONT]] ] +; CHECK-NEXT: ret float [[I9]] ; bb: %i = alloca float, align 4