diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h --- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h +++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h @@ -618,6 +618,7 @@ Instruction *foldPHIArgOpIntoPHI(PHINode &PN); Instruction *foldPHIArgBinOpIntoPHI(PHINode &PN); Instruction *foldPHIArgInsertValueInstructionIntoPHI(PHINode &PN); + Instruction *foldPHIArgExtractValueInstructionIntoPHI(PHINode &PN); Instruction *foldPHIArgGEPIntoPHI(PHINode &PN); Instruction *foldPHIArgLoadIntoPHI(PHINode &PN); Instruction *foldPHIArgZextsIntoPHI(PHINode &PN); diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp @@ -32,6 +32,8 @@ STATISTIC(NumPHIsOfInsertValues, "Number of phi-of-insertvalue turned into insertvalue-of-phis"); +STATISTIC(NumPHIsOfExtractValues, + "Number of phi-of-extractvalue turned into extractvalue-of-phi"); /// The PHI arguments will be folded into a single operation with a PHI node /// as input. The debug location of the single operation will be the merged @@ -336,6 +338,41 @@ return NewIVI; } +/// If we have something like phi [extractvalue(a,0), extractvalue(b,0)], +/// turn this into a phi[a,b] and a single extractvalue. +Instruction * +InstCombinerImpl::foldPHIArgExtractValueInstructionIntoPHI(PHINode &PN) { + auto *FirstEVI = cast(PN.getIncomingValue(0)); + + // Scan to see if all operands are `extractvalue`'s with the same indicies, + // and all have a single use. + for (unsigned i = 1; i != PN.getNumIncomingValues(); ++i) { + auto *I = dyn_cast(PN.getIncomingValue(i)); + if (!I || !I->hasOneUse() || I->getIndices() != FirstEVI->getIndices()) + return nullptr; + } + + // Create a new PHI node to receive the values the aggregate operand has + // in each incoming basic block. + auto *NewAggregateOperand = PHINode::Create( + FirstEVI->getAggregateOperand()->getType(), PN.getNumIncomingValues(), + FirstEVI->getAggregateOperand()->getName() + ".pn"); + // And populate the PHI with said values. + for (auto Incoming : zip(PN.blocks(), PN.incoming_values())) + NewAggregateOperand->addIncoming( + cast(std::get<1>(Incoming))->getAggregateOperand(), + std::get<0>(Incoming)); + InsertNewInstBefore(NewAggregateOperand, PN); + + // And finally, create `extractvalue` over the newly-formed PHI nodes. + auto *NewEVI = ExtractValueInst::Create(NewAggregateOperand, + FirstEVI->getIndices(), PN.getName()); + + PHIArgMergedDebugLoc(NewEVI, PN); + ++NumPHIsOfExtractValues; + return NewEVI; +} + /// If we have something like phi [add (a,b), add(a,c)] and if a/b/c and the /// adds all have a single use, turn this into a phi and a single binop. Instruction *InstCombinerImpl::foldPHIArgBinOpIntoPHI(PHINode &PN) { @@ -788,6 +825,8 @@ return foldPHIArgLoadIntoPHI(PN); if (isa(FirstInst)) return foldPHIArgInsertValueInstructionIntoPHI(PN); + if (isa(FirstInst)) + return foldPHIArgExtractValueInstructionIntoPHI(PN); // Scan the instruction, looking for input operations that can be folded away. // If all input operands to the phi are the same instruction (e.g. a cast from diff --git a/llvm/test/Transforms/InstCombine/phi-aware-aggregate-reconstruction.ll b/llvm/test/Transforms/InstCombine/phi-aware-aggregate-reconstruction.ll --- a/llvm/test/Transforms/InstCombine/phi-aware-aggregate-reconstruction.ll +++ b/llvm/test/Transforms/InstCombine/phi-aware-aggregate-reconstruction.ll @@ -24,9 +24,14 @@ ; CHECK-NEXT: call void @bar() ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[I8_MERGED:%.*]] = phi { i32, i32 } [ [[AGG_RIGHT:%.*]], [[RIGHT]] ], [ [[AGG_LEFT:%.*]], [[LEFT]] ] +; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] +; CHECK-NEXT: [[AGG_LEFT_PN1:%.*]] = phi { i32, i32 } [ [[AGG_LEFT]], [[LEFT]] ], [ [[AGG_RIGHT]], [[RIGHT]] ] +; CHECK-NEXT: [[I6:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT_PN1]], 1 +; CHECK-NEXT: [[I5:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT_PN]], 0 ; CHECK-NEXT: call void @baz() -; CHECK-NEXT: ret { i32, i32 } [[I8_MERGED]] +; CHECK-NEXT: [[I7:%.*]] = insertvalue { i32, i32 } undef, i32 [[I5]], 0 +; CHECK-NEXT: [[I8:%.*]] = insertvalue { i32, i32 } [[I7]], i32 [[I6]], 1 +; CHECK-NEXT: ret { i32, i32 } [[I8]] ; entry: br i1 %c, label %left, label %right @@ -58,18 +63,16 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 [[C:%.*]], label [[LEFT:%.*]], label [[RIGHT:%.*]] ; CHECK: left: -; CHECK-NEXT: [[I4:%.*]] = extractvalue { i32, i32 } [[AGG_RIGHT:%.*]], 1 -; CHECK-NEXT: [[I0:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT:%.*]], 0 ; CHECK-NEXT: call void @foo() ; CHECK-NEXT: br label [[END:%.*]] ; CHECK: right: -; CHECK-NEXT: [[I3:%.*]] = extractvalue { i32, i32 } [[AGG_RIGHT]], 0 -; CHECK-NEXT: [[I2:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT]], 1 ; CHECK-NEXT: call void @bar() ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[I5:%.*]] = phi i32 [ [[I0]], [[LEFT]] ], [ [[I3]], [[RIGHT]] ] -; CHECK-NEXT: [[I6:%.*]] = phi i32 [ [[I4]], [[LEFT]] ], [ [[I2]], [[RIGHT]] ] +; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] +; CHECK-NEXT: [[AGG_RIGHT_PN:%.*]] = phi { i32, i32 } [ [[AGG_RIGHT]], [[LEFT]] ], [ [[AGG_LEFT]], [[RIGHT]] ] +; CHECK-NEXT: [[I6:%.*]] = extractvalue { i32, i32 } [[AGG_RIGHT_PN]], 1 +; CHECK-NEXT: [[I5:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT_PN]], 0 ; CHECK-NEXT: call void @baz() ; CHECK-NEXT: [[I7:%.*]] = insertvalue { i32, i32 } undef, i32 [[I5]], 0 ; CHECK-NEXT: [[I8:%.*]] = insertvalue { i32, i32 } [[I7]], i32 [[I6]], 1 @@ -154,24 +157,20 @@ ; CHECK: bb0.dispatch: ; CHECK-NEXT: br i1 [[C1:%.*]], label [[BB00:%.*]], label [[BB01:%.*]] ; CHECK: bb00: -; CHECK-NEXT: [[I0:%.*]] = extractvalue { i32, i32 } [[AGG_00:%.*]], 0 -; CHECK-NEXT: [[I1:%.*]] = extractvalue { i32, i32 } [[AGG_00]], 1 ; CHECK-NEXT: br label [[BB0_MERGE:%.*]] ; CHECK: bb01: -; CHECK-NEXT: [[I2:%.*]] = extractvalue { i32, i32 } [[AGG_01:%.*]], 0 -; CHECK-NEXT: [[I3:%.*]] = extractvalue { i32, i32 } [[AGG_01]], 1 ; CHECK-NEXT: br label [[BB0_MERGE]] ; CHECK: bb0.merge: -; CHECK-NEXT: [[I4:%.*]] = phi i32 [ [[I0]], [[BB00]] ], [ [[I2]], [[BB01]] ] -; CHECK-NEXT: [[I5:%.*]] = phi i32 [ [[I1]], [[BB00]] ], [ [[I3]], [[BB01]] ] +; CHECK-NEXT: [[AGG_00_PN:%.*]] = phi { i32, i32 } [ [[AGG_00:%.*]], [[BB00]] ], [ [[AGG_01:%.*]], [[BB01]] ] +; CHECK-NEXT: [[AGG_00_PN1:%.*]] = phi { i32, i32 } [ [[AGG_00]], [[BB00]] ], [ [[AGG_01]], [[BB01]] ] ; CHECK-NEXT: br label [[END:%.*]] ; CHECK: bb10: -; CHECK-NEXT: [[I6:%.*]] = extractvalue { i32, i32 } [[AGG_10:%.*]], 0 -; CHECK-NEXT: [[I7:%.*]] = extractvalue { i32, i32 } [[AGG_10]], 1 ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[I8:%.*]] = phi i32 [ [[I4]], [[BB0_MERGE]] ], [ [[I6]], [[BB10]] ] -; CHECK-NEXT: [[I9:%.*]] = phi i32 [ [[I5]], [[BB0_MERGE]] ], [ [[I7]], [[BB10]] ] +; CHECK-NEXT: [[AGG_00_PN_PN:%.*]] = phi { i32, i32 } [ [[AGG_00_PN]], [[BB0_MERGE]] ], [ [[AGG_10:%.*]], [[BB10]] ] +; CHECK-NEXT: [[AGG_00_PN1_PN:%.*]] = phi { i32, i32 } [ [[AGG_00_PN1]], [[BB0_MERGE]] ], [ [[AGG_10]], [[BB10]] ] +; CHECK-NEXT: [[I9:%.*]] = extractvalue { i32, i32 } [[AGG_00_PN1_PN]], 1 +; CHECK-NEXT: [[I8:%.*]] = extractvalue { i32, i32 } [[AGG_00_PN_PN]], 0 ; CHECK-NEXT: call void @baz() ; CHECK-NEXT: [[I10:%.*]] = insertvalue { i32, i32 } undef, i32 [[I8]], 0 ; CHECK-NEXT: [[I11:%.*]] = insertvalue { i32, i32 } [[I10]], i32 [[I9]], 1 @@ -278,12 +277,17 @@ ; CHECK-NEXT: call void @bar() ; CHECK-NEXT: br label [[MIDDLE]] ; CHECK: middle: -; CHECK-NEXT: [[I8_MERGED:%.*]] = phi { i32, i32 } [ [[I8_MERGED]], [[MIDDLE]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ], [ [[AGG_LEFT:%.*]], [[LEFT]] ] +; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ], [ [[I8:%.*]], [[MIDDLE]] ] +; CHECK-NEXT: [[AGG_LEFT_PN1:%.*]] = phi { i32, i32 } [ [[AGG_LEFT]], [[LEFT]] ], [ [[AGG_RIGHT]], [[RIGHT]] ], [ [[I8]], [[MIDDLE]] ] +; CHECK-NEXT: [[I6:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT_PN1]], 1 +; CHECK-NEXT: [[I5:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT_PN]], 0 ; CHECK-NEXT: call void @baz() +; CHECK-NEXT: [[I7:%.*]] = insertvalue { i32, i32 } undef, i32 [[I5]], 0 +; CHECK-NEXT: [[I8]] = insertvalue { i32, i32 } [[I7]], i32 [[I6]], 1 ; CHECK-NEXT: [[C1:%.*]] = call i1 @geni1() ; CHECK-NEXT: br i1 [[C1]], label [[END:%.*]], label [[MIDDLE]] ; CHECK: end: -; CHECK-NEXT: ret { i32, i32 } [[I8_MERGED]] +; CHECK-NEXT: ret { i32, i32 } [[I8]] ; entry: br i1 %c0, label %left, label %right @@ -327,15 +331,20 @@ ; CHECK-NEXT: call void @bar() ; CHECK-NEXT: br label [[MERGE]] ; CHECK: merge: -; CHECK-NEXT: [[I8_MERGED:%.*]] = phi { i32, i32 } [ [[AGG_RIGHT:%.*]], [[RIGHT]] ], [ [[AGG_LEFT:%.*]], [[LEFT]] ] +; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] +; CHECK-NEXT: [[AGG_LEFT_PN1:%.*]] = phi { i32, i32 } [ [[AGG_LEFT]], [[LEFT]] ], [ [[AGG_RIGHT]], [[RIGHT]] ] ; CHECK-NEXT: call void @baz() ; CHECK-NEXT: br i1 [[C1:%.*]], label [[END:%.*]], label [[PASSTHROUGH:%.*]] ; CHECK: passthrough: ; CHECK-NEXT: call void @qux() ; CHECK-NEXT: br label [[END]] ; CHECK: end: +; CHECK-NEXT: [[I6:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT_PN1]], 1 +; CHECK-NEXT: [[I5:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT_PN]], 0 ; CHECK-NEXT: call void @quux() -; CHECK-NEXT: ret { i32, i32 } [[I8_MERGED]] +; CHECK-NEXT: [[I7:%.*]] = insertvalue { i32, i32 } undef, i32 [[I5]], 0 +; CHECK-NEXT: [[I8:%.*]] = insertvalue { i32, i32 } [[I7]], i32 [[I6]], 1 +; CHECK-NEXT: ret { i32, i32 } [[I8]] ; entry: br i1 %c0, label %left, label %right @@ -490,9 +499,14 @@ ; CHECK-NEXT: call void @bar() ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[I8_MERGED:%.*]] = phi { i32, i32 } [ [[AGG_RIGHT:%.*]], [[RIGHT]] ], [ [[AGG_LEFT:%.*]], [[LEFT]] ] +; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] +; CHECK-NEXT: [[AGG_LEFT_PN1:%.*]] = phi { i32, i32 } [ [[AGG_LEFT]], [[LEFT]] ], [ [[AGG_RIGHT]], [[RIGHT]] ] +; CHECK-NEXT: [[I7:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT_PN1]], 1 +; CHECK-NEXT: [[I0_PN:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT_PN]], 0 +; CHECK-NEXT: [[I6:%.*]] = insertvalue { i32, i32 } undef, i32 [[I0_PN]], 0 ; CHECK-NEXT: call void @baz() -; CHECK-NEXT: ret { i32, i32 } [[I8_MERGED]] +; CHECK-NEXT: [[I8:%.*]] = insertvalue { i32, i32 } [[I6]], i32 [[I7]], 1 +; CHECK-NEXT: ret { i32, i32 } [[I8]] ; entry: br i1 %c, label %left, label %right diff --git a/llvm/test/Transforms/InstCombine/phi-of-extractvalues.ll b/llvm/test/Transforms/InstCombine/phi-of-extractvalues.ll --- a/llvm/test/Transforms/InstCombine/phi-of-extractvalues.ll +++ b/llvm/test/Transforms/InstCombine/phi-of-extractvalues.ll @@ -10,13 +10,12 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 [[C:%.*]], label [[LEFT:%.*]], label [[RIGHT:%.*]] ; CHECK: left: -; CHECK-NEXT: [[I0:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT:%.*]], 0 ; CHECK-NEXT: br label [[END:%.*]] ; CHECK: right: -; CHECK-NEXT: [[I1:%.*]] = extractvalue { i32, i32 } [[AGG_RIGHT:%.*]], 0 ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[I0]], [[LEFT]] ], [ [[I1]], [[RIGHT]] ] +; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { i32, i32 } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] +; CHECK-NEXT: [[R:%.*]] = extractvalue { i32, i32 } [[AGG_LEFT_PN]], 0 ; CHECK-NEXT: ret i32 [[R]] ; entry: @@ -169,13 +168,12 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 [[C:%.*]], label [[LEFT:%.*]], label [[RIGHT:%.*]] ; CHECK: left: -; CHECK-NEXT: [[I0:%.*]] = extractvalue { { i32, i32 }, { i32, i32 } } [[AGG_LEFT:%.*]], 0, 0 ; CHECK-NEXT: br label [[END:%.*]] ; CHECK: right: -; CHECK-NEXT: [[I1:%.*]] = extractvalue { { i32, i32 }, { i32, i32 } } [[AGG_RIGHT:%.*]], 0, 0 ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[I0]], [[LEFT]] ], [ [[I1]], [[RIGHT]] ] +; CHECK-NEXT: [[AGG_LEFT_PN:%.*]] = phi { { i32, i32 }, { i32, i32 } } [ [[AGG_LEFT:%.*]], [[LEFT]] ], [ [[AGG_RIGHT:%.*]], [[RIGHT]] ] +; CHECK-NEXT: [[R:%.*]] = extractvalue { { i32, i32 }, { i32, i32 } } [[AGG_LEFT_PN]], 0, 0 ; CHECK-NEXT: ret i32 [[R]] ; entry: