diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "InstCombineInternal.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/Statistic.h" @@ -340,38 +341,218 @@ } /// If we have something like phi [extractvalue(a,0), extractvalue(b,0)], -/// turn this into a phi[a,b] and a single extractvalue. +/// or phi [phi [extractvalue(a,0), extractvalue(b,0)], extractvalue(c,0)] +/// turn this into a phi[a,b] / phi[phi[a,b],c] and a single extractvalue. Instruction * -InstCombinerImpl::foldPHIArgExtractValueInstructionIntoPHI(PHINode &PN) { - auto *FirstEVI = cast(PN.getIncomingValue(0)); - - // Scan to see if all operands are `extractvalue`'s with the same indicies, - // and all have a single use. - for (unsigned i = 1; i != PN.getNumIncomingValues(); ++i) { - auto *I = dyn_cast(PN.getIncomingValue(i)); - if (!I || !I->hasOneUser() || I->getIndices() != FirstEVI->getIndices() || - I->getAggregateOperand()->getType() != - FirstEVI->getAggregateOperand()->getType()) +InstCombinerImpl::foldPHIArgExtractValueInstructionIntoPHI(PHINode &RootPN) { + // We cannot create a new instruction after the PHI if the terminator is an + // EHPad because there is no valid insertion point. + if (Instruction *TI = RootPN.getParent()->getTerminator()) + if (TI->isEHPad()) return nullptr; + + // The reference def. All other defs must be compatible with this one, + // else we won't be able to perform the transformation. + Optional FirstEVI; + + struct NodeDsc { + // How many uses does this node has within the graph? + // Note that there may be out-of-graph uses! + unsigned NumUsesSeen = 0; + // After remapping, what new value shall be used instead of this one? + Value *NewV = nullptr; + }; + + // A map of all the graph nodes. + // WARNING: this must have stable iteration order! + constexpr unsigned NodeCountSmallSize = 64; // 94'th percentile + SmallMapVector Graph; + + // Potentially record the node in a map, and bump it's observed use count. + // Returns whether or not this node is encountered for the first time. + auto RecordNode = [&Graph](Value *V) { + // See if we've already seen this Value or record a new encountered Value. + std::pair I = + Graph.insert({V, {}}); + // Regardless, record that we have reached the node it from a new use. + ++I.first->second.NumUsesSeen; + return I.second; // Is this the first time we've seen this node? + }; + + // A data structure to keep track of which nodes we still need to visit, + // during initial graph prospection. The order in which we'll process the + // nodes doesn't really matter, but let's do a classic depth-first traversal. + constexpr unsigned WorklistSmallSize = 8; // 98'th percentile + SmallVector Worklist; + + // Given this PHI, potentially enqueue it for further analysis. + // NOTE: we don't/can't do any use-checking here! + // We must first process the entire graph! + auto EnqueuePHI = [&RecordNode, &Worklist](PHINode *PN) { + // See if this Value was seen already and/or record a new encountered Value. + if (!RecordNode(PN)) + return; // Node previously seen. No further action needed. + // Otherwise, actually enqueue it for analysis. + Worklist.emplace_back(PN); + }; + + // Can we PHI together operands of (all the) extractvalue`s? + auto DefsAreCompatible = [](ExtractValueInst *A, ExtractValueInst *B) { + return A->getAggregateOperand()->getType() == + B->getAggregateOperand()->getType() && + A->getIndices() == B->getIndices(); + }; + + // Given this value, which is not a PHI and therefore must be a def, + // potentially record it as a new graph node, and see if it's an + // extractvalue that is compatible with previous defs we've reached. + // NOTE: we don't/can't do any use-checking here! + // We must first process the entire graph! + auto RecordDef = [&RecordNode, &FirstEVI, + &DefsAreCompatible](Value *V) -> bool /*IsCompatibleDef*/ { + auto *EVI = dyn_cast(V); + if (!EVI) + return false; // Not an extractvalue instruction. Give up. + + // See if this Value was seen already and/or record a new encountered Value. + if (!RecordNode(EVI)) + return true; // First time seeing. All good, no further checking needed. + + // Is this the first extractvalue we've found? + if (!FirstEVI) { + FirstEVI = EVI; + return true; // All good, no further checking possible. + } + + // Otherwise, if this is the first time we're seeing this extractvalue, + // but not the first extractvalue we're seeing, we actually do need to + // check that they're compatible. + return DefsAreCompatible(EVI, *FirstEVI); + }; + + // Traverse this PHI recursively, and for each incoming value, + // either recurse into the new PHI, or record the defs, + // all of which must be compatible extractvalue's. + EnqueuePHI(&RootPN); + while (!Worklist.empty()) { + PHINode *PN = Worklist.pop_back_val(); + + // Now, let's see what this PHI consists of, what are it's incoming values. + for (Value *V : PN->incoming_values()) { + // Enqueue each PHI node to be visited in later iteration of the loop. + if (auto *PN = dyn_cast(V)) { + EnqueuePHI(PN); // Updates Worklist. + continue; // Onto next incoming value. + } + + // Otherwise, if it's not a PHI node, it must be a def. Is it a good def? + if (!RecordDef(V)) + return nullptr; // Bad def, abort. + // Onto next yet-unexplored graph node. + } + } + assert(Graph.size() >= 2 && + "Should have mapped a graph with at least two nodes."); + assert(FirstEVI && "Should've found at least one extractvalue instruction."); + + // Okay! The original PHI RootPN, recursively, consists only of other PHI's + // and intercompatible extractvalue`s! So we can pull them through the PHI's. + + // One thing though, we need to ensure that the transform does not increase + // instruction count. Reminder: we have N PHI nodes and M extractvalue`s, + // we'll need to produce N PHI nodes, and a single extractvalue. + // Therefore, all old PHI nodes and at least one extractvalue must go away. + + auto GraphNodeHasNoOutOfGraphUses = [&RootPN, &Graph]( + std::pair Node) { + assert(Node.first != &RootPN && "We shouldn't use-check the root PHI node"); + + // We've counted how many times each graph node is used by other graph nodes + // so to check that the node has no out-of-graph uses we only need to check + // that the Value use count matches the number of in-graph uses. + bool AllUsesSeen = Node.first->hasNUses(Node.second.NumUsesSeen); + + (void)Graph; + assert(all_of(Node.first->users(), + [&Graph](User *U) { + return isa(U) && Graph.count(U); + }) == AllUsesSeen && + "Sanity check: checking whether or not the node has uses that are " + "not recorded in the Graph map should be identical to checking the " + "recorded numer of times we've reached the value with use count of " + "the value."); + + return AllUsesSeen; + }; + + // At least one extractvalue must only have in-graph uses, and go away. + if (none_of(make_filter_range(Graph, + [](std::pair Node) { + return !isa(Node.first); + }), + GraphNodeHasNoOutOfGraphUses)) + return nullptr; + + // And all the PHI nodes must only have in-graph ses and thus go away. + assert(Graph.front().first == &RootPN && + "Expected the root PHI to be the first element in Graph mapvector."); + if (!all_of(make_filter_range(ArrayRef>( + Graph.begin(), Graph.end()) + .drop_front(), + [](std::pair Node) { + return isa(Node.first); + }), + GraphNodeHasNoOutOfGraphUses)) + return nullptr; + + // Awesome. We are committed to a rewrite now. + // The actual rewrite consists of three separate steps. + + // Rewrite step 1: remap each graph node, separately, but don't recreate graph + // just yet (because we are not visiting nodes for remapping in def-use order, + // so it is not guaranteed that all defs were remapped already). + // NOTE: the iteration order *IS* (and must be) stable. + Type *AggTy = (*FirstEVI)->getAggregateOperand()->getType(); + for (std::pair &Node : Graph) { + Value *OldV = Node.first; + Value *&NewV = Node.second.NewV; + + if (auto *OldPN = dyn_cast(OldV)) { + NewV = PHINode::Create(AggTy, OldPN->getNumIncomingValues(), + OldPN->getName() + ".agg"); + InsertNewInstBefore(cast(NewV), *OldPN); + } else + NewV = cast(OldV)->getAggregateOperand(); } - // Create a new PHI node to receive the values the aggregate operand has - // in each incoming basic block. - auto *NewAggregateOperand = PHINode::Create( - FirstEVI->getAggregateOperand()->getType(), PN.getNumIncomingValues(), - FirstEVI->getAggregateOperand()->getName() + ".pn"); - // And populate the PHI with said values. - for (auto Incoming : zip(PN.blocks(), PN.incoming_values())) - NewAggregateOperand->addIncoming( - cast(std::get<1>(Incoming))->getAggregateOperand(), - std::get<0>(Incoming)); - InsertNewInstBefore(NewAggregateOperand, PN); + auto GetRemappedValue = [&Graph](Value *OldV) { + auto I = Graph.find(OldV); + assert(I != Graph.end() && "Not a graph node?"); + Value *NewV = I->second.NewV; + assert(NewV && "Node should have been remaped by now."); + return NewV; + }; - // And finally, create `extractvalue` over the newly-formed PHI nodes. - auto *NewEVI = ExtractValueInst::Create(NewAggregateOperand, - FirstEVI->getIndices(), PN.getName()); + // Rewrite step 2: actually recreate the def-use graph. + // Now that all the nodes were remapped in step 1, we can actually populate + // PHI nodes with proper remapped values. + // NOTE: the iteration order is not externally obserable, but is stable. + for (const std::pair &Node : Graph) { + auto *OldPN = dyn_cast(Node.first); + if (!OldPN) + continue; // Only have to operate on PHI's now. + auto *NewPN = cast(Node.second.NewV); + // Populate this new PHI node with remapped incoming values. + for (auto Operands : zip(OldPN->incoming_values(), OldPN->blocks())) + NewPN->addIncoming(GetRemappedValue(std::get<0>(Operands)), + std::get<1>(Operands)); + } - PHIArgMergedDebugLoc(NewEVI, PN); + // Rewrite step 3: create the extractvalue over the remapped root PHI node. + auto *NewEVI = ExtractValueInst::Create( + GetRemappedValue(&RootPN), (*FirstEVI)->getIndices(), RootPN.getName()); + + PHIArgMergedDebugLoc(NewEVI, RootPN); ++NumPHIsOfExtractValues; return NewEVI; } @@ -829,8 +1010,6 @@ return foldPHIArgLoadIntoPHI(PN); if (isa(FirstInst)) return foldPHIArgInsertValueInstructionIntoPHI(PN); - if (isa(FirstInst)) - return foldPHIArgExtractValueInstructionIntoPHI(PN); // Scan the instruction, looking for input operations that can be folded away. // If all input operands to the phi are the same instruction (e.g. a cast from @@ -1310,6 +1489,10 @@ if (Instruction *Result = foldPHIArgOpIntoPHI(PN)) return Result; + // If all non-PHI PHI operands are extractvalues, pull them through the PHI's. + if (Instruction *Result = foldPHIArgExtractValueInstructionIntoPHI(PN)) + return Result; + // If this is a trivial cycle in the PHI node graph, remove it. Basically, if // this PHI only has a single use (a PHI), and if that PHI only has one use (a // PHI)... break the cycle. diff --git a/llvm/test/Transforms/InstCombine/phi-aware-aggregate-reconstruction.ll b/llvm/test/Transforms/InstCombine/phi-aware-aggregate-reconstruction.ll --- a/llvm/test/Transforms/InstCombine/phi-aware-aggregate-reconstruction.ll +++ b/llvm/test/Transforms/InstCombine/phi-aware-aggregate-reconstruction.ll @@ -24,9 +24,9 @@ ; CHECK-NEXT: call void @bar() ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] +; CHECK-NEXT: [[I5_AGG:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] ; CHECK-NEXT: call void @baz() -; CHECK-NEXT: ret { i32, i32 } [[AGG_LEFT_PN]] +; CHECK-NEXT: ret { i32, i32 } [[I5_AGG]] ; entry: br i1 %c, label %left, label %right @@ -64,10 +64,10 @@ ; CHECK-NEXT: call void @bar() ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] -; CHECK-NEXT: [[AGG_RIGHT_PN:%.*]] = phi { i32, i32 } [ [[AGG_RIGHT]], [[LEFT]] ], [ [[AGG_LEFT]], [[RIGHT]] ] -; CHECK-NEXT: [[I6:%.*]] = extractvalue { i32, i32 } [[AGG_RIGHT_PN]], 1 -; CHECK-NEXT: [[I5:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT_PN]], 0 +; CHECK-NEXT: [[I5_AGG:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] +; CHECK-NEXT: [[I6_AGG:%.*]] = phi { i32, i32 } [ [[AGG_RIGHT]], [[LEFT]] ], [ [[AGG_LEFT]], [[RIGHT]] ] +; CHECK-NEXT: [[I6:%.*]] = extractvalue { i32, i32 } [[I6_AGG]], 1 +; CHECK-NEXT: [[I5:%.*]] = extractvalue { i32, i32 } [[I5_AGG]], 0 ; CHECK-NEXT: call void @baz() ; CHECK-NEXT: [[I7:%.*]] = insertvalue { i32, i32 } undef, i32 [[I5]], 0 ; CHECK-NEXT: [[I8:%.*]] = insertvalue { i32, i32 } [[I7]], i32 [[I6]], 1 @@ -156,14 +156,14 @@ ; CHECK: bb01: ; CHECK-NEXT: br label [[BB0_MERGE]] ; CHECK: bb0.merge: -; CHECK-NEXT: [[AGG_00_PN:%.*]] = phi { i32, i32 } [ [[AGG_00:%.*]], [[BB00]] ], [ [[AGG_01:%.*]], [[BB01]] ] +; CHECK-NEXT: [[I4_AGG:%.*]] = phi { i32, i32 } [ [[AGG_00:%.*]], [[BB00]] ], [ [[AGG_01:%.*]], [[BB01]] ] ; CHECK-NEXT: br label [[END:%.*]] ; CHECK: bb10: ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[AGG_00_PN_PN:%.*]] = phi { i32, i32 } [ [[AGG_00_PN]], [[BB0_MERGE]] ], [ [[AGG_10:%.*]], [[BB10]] ] +; CHECK-NEXT: [[I8_AGG:%.*]] = phi { i32, i32 } [ [[I4_AGG]], [[BB0_MERGE]] ], [ [[AGG_10:%.*]], [[BB10]] ] ; CHECK-NEXT: call void @baz() -; CHECK-NEXT: ret { i32, i32 } [[AGG_00_PN_PN]] +; CHECK-NEXT: ret { i32, i32 } [[I8_AGG]] ; entry: br i1 %c0, label %bb0.dispatch, label %bb10 @@ -206,23 +206,21 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 [[C0:%.*]], label [[LEFT:%.*]], label [[RIGHT:%.*]] ; CHECK: left: -; CHECK-NEXT: [[I0:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT:%.*]], 0 -; CHECK-NEXT: [[I2:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT]], 1 ; CHECK-NEXT: call void @foo() ; CHECK-NEXT: br label [[MIDDLE:%.*]] ; CHECK: right: -; CHECK-NEXT: [[I3:%.*]] = extractvalue { i32, i32 } [[AGG_RIGHT:%.*]], 0 -; CHECK-NEXT: [[I4:%.*]] = extractvalue { i32, i32 } [[AGG_RIGHT]], 1 ; CHECK-NEXT: call void @bar() ; CHECK-NEXT: br label [[MIDDLE]] ; CHECK: middle: -; CHECK-NEXT: [[I5:%.*]] = phi i32 [ [[I0]], [[LEFT]] ], [ [[I3]], [[RIGHT]] ], [ [[I5]], [[MIDDLE]] ] -; CHECK-NEXT: [[I6:%.*]] = phi i32 [ [[I2]], [[LEFT]] ], [ [[I4]], [[RIGHT]] ], [ [[I6]], [[MIDDLE]] ] +; CHECK-NEXT: [[I5_AGG:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ], [ [[I5_AGG]], [[MIDDLE]] ] +; CHECK-NEXT: [[I6_AGG:%.*]] = phi { i32, i32 } [ [[AGG_LEFT]], [[LEFT]] ], [ [[AGG_RIGHT]], [[RIGHT]] ], [ [[I6_AGG]], [[MIDDLE]] ] ; CHECK-NEXT: call void @baz() ; CHECK-NEXT: [[C1:%.*]] = call i1 @geni1() ; CHECK-NEXT: br i1 [[C1]], label [[END:%.*]], label [[MIDDLE]] ; CHECK: end: +; CHECK-NEXT: [[I5:%.*]] = extractvalue { i32, i32 } [[I5_AGG]], 0 ; CHECK-NEXT: [[I7:%.*]] = insertvalue { i32, i32 } undef, i32 [[I5]], 0 +; CHECK-NEXT: [[I6:%.*]] = extractvalue { i32, i32 } [[I6_AGG]], 1 ; CHECK-NEXT: [[I8:%.*]] = insertvalue { i32, i32 } [[I7]], i32 [[I6]], 1 ; CHECK-NEXT: ret { i32, i32 } [[I8]] ; @@ -266,12 +264,12 @@ ; CHECK-NEXT: call void @bar() ; CHECK-NEXT: br label [[MIDDLE]] ; CHECK: middle: -; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ], [ [[AGG_LEFT_PN]], [[MIDDLE]] ] +; CHECK-NEXT: [[I5_AGG:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ], [ [[I5_AGG]], [[MIDDLE]] ] ; CHECK-NEXT: call void @baz() ; CHECK-NEXT: [[C1:%.*]] = call i1 @geni1() ; CHECK-NEXT: br i1 [[C1]], label [[END:%.*]], label [[MIDDLE]] ; CHECK: end: -; CHECK-NEXT: ret { i32, i32 } [[AGG_LEFT_PN]] +; CHECK-NEXT: ret { i32, i32 } [[I5_AGG]] ; entry: br i1 %c0, label %left, label %right @@ -315,7 +313,7 @@ ; CHECK-NEXT: call void @bar() ; CHECK-NEXT: br label [[MERGE]] ; CHECK: merge: -; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] +; CHECK-NEXT: [[I5_AGG:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] ; CHECK-NEXT: call void @baz() ; CHECK-NEXT: br i1 [[C1:%.*]], label [[END:%.*]], label [[PASSTHROUGH:%.*]] ; CHECK: passthrough: @@ -323,7 +321,7 @@ ; CHECK-NEXT: br label [[END]] ; CHECK: end: ; CHECK-NEXT: call void @quux() -; CHECK-NEXT: ret { i32, i32 } [[AGG_LEFT_PN]] +; CHECK-NEXT: ret { i32, i32 } [[I5_AGG]] ; entry: br i1 %c0, label %left, label %right @@ -429,9 +427,9 @@ ; CHECK: impossible: ; CHECK-NEXT: unreachable ; CHECK: end: -; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_LEFT]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ], [ [[AGG_RIGHT]], [[RIGHT]] ] +; CHECK-NEXT: [[I5_AGG:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_LEFT]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ], [ [[AGG_RIGHT]], [[RIGHT]] ] ; CHECK-NEXT: call void @baz() -; CHECK-NEXT: ret { i32, i32 } [[AGG_LEFT_PN]] +; CHECK-NEXT: ret { i32, i32 } [[I5_AGG]] ; entry: br i1 %c, label %left, label %right @@ -478,9 +476,9 @@ ; CHECK-NEXT: call void @bar() ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] +; CHECK-NEXT: [[I0_PN_AGG:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] ; CHECK-NEXT: call void @baz() -; CHECK-NEXT: ret { i32, i32 } [[AGG_LEFT_PN]] +; CHECK-NEXT: ret { i32, i32 } [[I0_PN_AGG]] ; entry: br i1 %c, label %left, label %right diff --git a/llvm/test/Transforms/InstCombine/phi-of-extractvalues.ll b/llvm/test/Transforms/InstCombine/phi-of-extractvalues.ll --- a/llvm/test/Transforms/InstCombine/phi-of-extractvalues.ll +++ b/llvm/test/Transforms/InstCombine/phi-of-extractvalues.ll @@ -14,8 +14,8 @@ ; CHECK: right: ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] -; CHECK-NEXT: [[R:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT_PN]], 0 +; CHECK-NEXT: [[R_AGG:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] +; CHECK-NEXT: [[R:%.*]] = extractvalue { i32, i32 } [[R_AGG]], 0 ; CHECK-NEXT: ret i32 [[R]] ; entry: @@ -172,8 +172,8 @@ ; CHECK: right: ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { { i32, i32 }, { i32, i32 } } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] -; CHECK-NEXT: [[R:%.*]] = extractvalue { { i32, i32 }, { i32, i32 } } [[AGG_LEFT_PN]], 0, 0 +; CHECK-NEXT: [[R_AGG:%.*]] = phi { { i32, i32 }, { i32, i32 } } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] +; CHECK-NEXT: [[R:%.*]] = extractvalue { { i32, i32 }, { i32, i32 } } [[R_AGG]], 0, 0 ; CHECK-NEXT: ret i32 [[R]] ; entry: @@ -325,8 +325,8 @@ ; CHECK: right: ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[ENTRY:%.*]] ], [ [[AGG_LEFT]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] -; CHECK-NEXT: [[R:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT_PN]], 0 +; CHECK-NEXT: [[R_AGG:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[ENTRY:%.*]] ], [ [[AGG_LEFT]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] +; CHECK-NEXT: [[R:%.*]] = extractvalue { i32, i32 } [[R_AGG]], 0 ; CHECK-NEXT: ret i32 [[R]] ; entry: