diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -19,6 +19,7 @@ #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/OverflowInstAnalysis.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constant.h" #include "llvm/IR/ConstantRange.h" @@ -2352,6 +2353,41 @@ } Instruction *InstCombinerImpl::foldVectorSelect(SelectInst &Sel) { + if (!isa(Sel.getType())) + return nullptr; + + Value *Cond = Sel.getCondition(); + Value *TVal = Sel.getTrueValue(); + Value *FVal = Sel.getFalseValue(); + Value *C, *X, *Y; + + if (match(Cond, m_VecReverse(m_Value(C)))) { + auto createSelReverse = [&](Value *C, Value *X, Value *Y) { + Value *V = Builder.CreateSelect(C, X, Y, Sel.getName(), &Sel); + if (auto *I = dyn_cast(V)) + I->copyIRFlags(&Sel); + Module *M = Sel.getModule(); + Function *F = Intrinsic::getDeclaration( + M, Intrinsic::experimental_vector_reverse, V->getType()); + return CallInst::Create(F, V); + }; + + if (match(TVal, m_VecReverse(m_Value(X)))) { + // select rev(C), rev(X), rev(Y) --> rev(select C, X, Y) + if (match(FVal, m_VecReverse(m_Value(Y))) && + (Cond->hasOneUse() || TVal->hasOneUse() || FVal->hasOneUse())) + return createSelReverse(C, X, Y); + + // select rev(C), rev(X), FValSplat --> rev(select C, X, FValSplat) + if ((Cond->hasOneUse() || TVal->hasOneUse()) && isSplatValue(FVal)) + return createSelReverse(C, X, FVal); + } + // select rev(C), TValSplat, rev(Y) --> rev(select C, TValSplat, Y) + else if (isSplatValue(TVal) && match(FVal, m_VecReverse(m_Value(Y))) && + (Cond->hasOneUse() || FVal->hasOneUse())) + return createSelReverse(C, TVal, Y); + } + auto *VecTy = dyn_cast(Sel.getType()); if (!VecTy) return nullptr; @@ -2368,10 +2404,6 @@ // A select of a "select shuffle" with a common operand can be rearranged // to select followed by "select shuffle". Because of poison, this only works // in the case of a shuffle with no undefined mask elements. - Value *Cond = Sel.getCondition(); - Value *TVal = Sel.getTrueValue(); - Value *FVal = Sel.getFalseValue(); - Value *X, *Y; ArrayRef Mask; if (match(TVal, m_OneUse(m_Shuffle(m_Value(X), m_Value(Y), m_Mask(Mask)))) && !is_contained(Mask, UndefMaskElem) && diff --git a/llvm/test/Transforms/InstCombine/vector-reverse.ll b/llvm/test/Transforms/InstCombine/vector-reverse.ll --- a/llvm/test/Transforms/InstCombine/vector-reverse.ll +++ b/llvm/test/Transforms/InstCombine/vector-reverse.ll @@ -301,10 +301,8 @@ define @select_reverse( %a, %b, %c) { ; CHECK-LABEL: @select_reverse( -; CHECK-NEXT: [[A_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i1( [[A:%.*]]) -; CHECK-NEXT: [[B_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[B:%.*]]) -; CHECK-NEXT: [[C_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[C:%.*]]) -; CHECK-NEXT: [[SELECT:%.*]] = select [[A_REV]], [[B_REV]], [[C_REV]] +; CHECK-NEXT: [[SELECT1:%.*]] = select [[A:%.*]], [[B:%.*]], [[C:%.*]] +; CHECK-NEXT: [[SELECT:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[SELECT1]]) ; CHECK-NEXT: ret [[SELECT]] ; %a.rev = tail call @llvm.experimental.vector.reverse.nxv4i1( %a) @@ -318,10 +316,9 @@ define @select_reverse_1( %a, %b, %c) { ; CHECK-LABEL: @select_reverse_1( ; CHECK-NEXT: [[A_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i1( [[A:%.*]]) -; CHECK-NEXT: [[B_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[B:%.*]]) -; CHECK-NEXT: [[C_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[C:%.*]]) ; CHECK-NEXT: call void @use_nxv4i1( [[A_REV]]) -; CHECK-NEXT: [[SELECT:%.*]] = select [[A_REV]], [[B_REV]], [[C_REV]] +; CHECK-NEXT: [[SELECT1:%.*]] = select [[A]], [[B:%.*]], [[C:%.*]] +; CHECK-NEXT: [[SELECT:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[SELECT1]]) ; CHECK-NEXT: ret [[SELECT]] ; %a.rev = tail call @llvm.experimental.vector.reverse.nxv4i1( %a) @@ -335,11 +332,10 @@ ; %b.rev has multiple uses define @select_reverse_2( %a, %b, %c) { ; CHECK-LABEL: @select_reverse_2( -; CHECK-NEXT: [[A_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i1( [[A:%.*]]) ; CHECK-NEXT: [[B_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[B:%.*]]) -; CHECK-NEXT: [[C_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[C:%.*]]) ; CHECK-NEXT: call void @use_nxv4i32( [[B_REV]]) -; CHECK-NEXT: [[SELECT:%.*]] = select [[A_REV]], [[B_REV]], [[C_REV]] +; CHECK-NEXT: [[SELECT1:%.*]] = select [[A:%.*]], [[B]], [[C:%.*]] +; CHECK-NEXT: [[SELECT:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[SELECT1]]) ; CHECK-NEXT: ret [[SELECT]] ; %a.rev = tail call @llvm.experimental.vector.reverse.nxv4i1( %a) @@ -353,11 +349,10 @@ ; %c.rev has multiple uses define @select_reverse_3( %a, %b, %c) { ; CHECK-LABEL: @select_reverse_3( -; CHECK-NEXT: [[A_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i1( [[A:%.*]]) -; CHECK-NEXT: [[B_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[B:%.*]]) ; CHECK-NEXT: [[C_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[C:%.*]]) ; CHECK-NEXT: call void @use_nxv4i32( [[C_REV]]) -; CHECK-NEXT: [[SELECT:%.*]] = select [[A_REV]], [[B_REV]], [[C_REV]] +; CHECK-NEXT: [[SELECT1:%.*]] = select [[A:%.*]], [[B:%.*]], [[C]] +; CHECK-NEXT: [[SELECT:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[SELECT1]]) ; CHECK-NEXT: ret [[SELECT]] ; %a.rev = tail call @llvm.experimental.vector.reverse.nxv4i1( %a) @@ -373,10 +368,10 @@ ; CHECK-LABEL: @select_reverse_4( ; CHECK-NEXT: [[A_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i1( [[A:%.*]]) ; CHECK-NEXT: [[B_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[B:%.*]]) -; CHECK-NEXT: [[C_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[C:%.*]]) ; CHECK-NEXT: call void @use_nxv4i1( [[A_REV]]) ; CHECK-NEXT: call void @use_nxv4i32( [[B_REV]]) -; CHECK-NEXT: [[SELECT:%.*]] = select [[A_REV]], [[B_REV]], [[C_REV]] +; CHECK-NEXT: [[SELECT1:%.*]] = select [[A]], [[B]], [[C:%.*]] +; CHECK-NEXT: [[SELECT:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[SELECT1]]) ; CHECK-NEXT: ret [[SELECT]] ; %a.rev = tail call @llvm.experimental.vector.reverse.nxv4i1( %a) @@ -392,11 +387,11 @@ define @select_reverse_5( %a, %b, %c) { ; CHECK-LABEL: @select_reverse_5( ; CHECK-NEXT: [[A_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i1( [[A:%.*]]) -; CHECK-NEXT: [[B_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[B:%.*]]) ; CHECK-NEXT: [[C_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[C:%.*]]) ; CHECK-NEXT: call void @use_nxv4i1( [[A_REV]]) ; CHECK-NEXT: call void @use_nxv4i32( [[C_REV]]) -; CHECK-NEXT: [[SELECT:%.*]] = select [[A_REV]], [[B_REV]], [[C_REV]] +; CHECK-NEXT: [[SELECT1:%.*]] = select [[A]], [[B:%.*]], [[C]] +; CHECK-NEXT: [[SELECT:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[SELECT1]]) ; CHECK-NEXT: ret [[SELECT]] ; %a.rev = tail call @llvm.experimental.vector.reverse.nxv4i1( %a) @@ -411,12 +406,12 @@ ; %b.rev and %c.rev have multiple uses define @select_reverse_6( %a, %b, %c) { ; CHECK-LABEL: @select_reverse_6( -; CHECK-NEXT: [[A_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i1( [[A:%.*]]) ; CHECK-NEXT: [[B_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[B:%.*]]) ; CHECK-NEXT: [[C_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[C:%.*]]) ; CHECK-NEXT: call void @use_nxv4i32( [[B_REV]]) ; CHECK-NEXT: call void @use_nxv4i32( [[C_REV]]) -; CHECK-NEXT: [[SELECT:%.*]] = select [[A_REV]], [[B_REV]], [[C_REV]] +; CHECK-NEXT: [[SELECT1:%.*]] = select [[A:%.*]], [[B]], [[C]] +; CHECK-NEXT: [[SELECT:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[SELECT1]]) ; CHECK-NEXT: ret [[SELECT]] ; %a.rev = tail call @llvm.experimental.vector.reverse.nxv4i1( %a) @@ -452,11 +447,10 @@ define @select_reverse_splat_false( %a, %b, i32 %c) { ; CHECK-LABEL: @select_reverse_splat_false( -; CHECK-NEXT: [[A_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i1( [[A:%.*]]) -; CHECK-NEXT: [[B_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[B:%.*]]) ; CHECK-NEXT: [[C_INSERT:%.*]] = insertelement poison, i32 [[C:%.*]], i64 0 ; CHECK-NEXT: [[C_SPLAT:%.*]] = shufflevector [[C_INSERT]], poison, zeroinitializer -; CHECK-NEXT: [[SELECT:%.*]] = select [[A_REV]], [[B_REV]], [[C_SPLAT]] +; CHECK-NEXT: [[SELECT1:%.*]] = select [[A:%.*]], [[B:%.*]], [[C_SPLAT]] +; CHECK-NEXT: [[SELECT:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[SELECT1]]) ; CHECK-NEXT: ret [[SELECT]] ; %a.rev = tail call @llvm.experimental.vector.reverse.nxv4i1( %a) @@ -471,11 +465,11 @@ define @select_reverse_splat_false_1( %a, %b, i32 %c) { ; CHECK-LABEL: @select_reverse_splat_false_1( ; CHECK-NEXT: [[A_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i1( [[A:%.*]]) -; CHECK-NEXT: [[B_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[B:%.*]]) ; CHECK-NEXT: [[C_INSERT:%.*]] = insertelement poison, i32 [[C:%.*]], i64 0 ; CHECK-NEXT: [[C_SPLAT:%.*]] = shufflevector [[C_INSERT]], poison, zeroinitializer ; CHECK-NEXT: call void @use_nxv4i1( [[A_REV]]) -; CHECK-NEXT: [[SELECT:%.*]] = select [[A_REV]], [[B_REV]], [[C_SPLAT]] +; CHECK-NEXT: [[SELECT1:%.*]] = select [[A]], [[B:%.*]], [[C_SPLAT]] +; CHECK-NEXT: [[SELECT:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[SELECT1]]) ; CHECK-NEXT: ret [[SELECT]] ; %a.rev = tail call @llvm.experimental.vector.reverse.nxv4i1( %a) @@ -490,12 +484,12 @@ ; %b.rev has multiple uses define @select_reverse_splat_false_2( %a, %b, i32 %c) { ; CHECK-LABEL: @select_reverse_splat_false_2( -; CHECK-NEXT: [[A_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i1( [[A:%.*]]) ; CHECK-NEXT: [[B_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[B:%.*]]) ; CHECK-NEXT: [[C_INSERT:%.*]] = insertelement poison, i32 [[C:%.*]], i64 0 ; CHECK-NEXT: [[C_SPLAT:%.*]] = shufflevector [[C_INSERT]], poison, zeroinitializer ; CHECK-NEXT: call void @use_nxv4i32( [[B_REV]]) -; CHECK-NEXT: [[SELECT:%.*]] = select [[A_REV]], [[B_REV]], [[C_SPLAT]] +; CHECK-NEXT: [[SELECT1:%.*]] = select [[A:%.*]], [[B]], [[C_SPLAT]] +; CHECK-NEXT: [[SELECT:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[SELECT1]]) ; CHECK-NEXT: ret [[SELECT]] ; %a.rev = tail call @llvm.experimental.vector.reverse.nxv4i1( %a) @@ -531,11 +525,10 @@ define @select_reverse_splat_true( %a, %b, i32 %c) { ; CHECK-LABEL: @select_reverse_splat_true( -; CHECK-NEXT: [[A_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i1( [[A:%.*]]) -; CHECK-NEXT: [[B_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[B:%.*]]) ; CHECK-NEXT: [[C_INSERT:%.*]] = insertelement poison, i32 [[C:%.*]], i64 0 ; CHECK-NEXT: [[C_SPLAT:%.*]] = shufflevector [[C_INSERT]], poison, zeroinitializer -; CHECK-NEXT: [[SELECT:%.*]] = select [[A_REV]], [[C_SPLAT]], [[B_REV]] +; CHECK-NEXT: [[SELECT1:%.*]] = select [[A:%.*]], [[C_SPLAT]], [[B:%.*]] +; CHECK-NEXT: [[SELECT:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[SELECT1]]) ; CHECK-NEXT: ret [[SELECT]] ; %a.rev = tail call @llvm.experimental.vector.reverse.nxv4i1( %a) @@ -550,11 +543,11 @@ define @select_reverse_splat_true_1( %a, %b, i32 %c) { ; CHECK-LABEL: @select_reverse_splat_true_1( ; CHECK-NEXT: [[A_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i1( [[A:%.*]]) -; CHECK-NEXT: [[B_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[B:%.*]]) ; CHECK-NEXT: [[C_INSERT:%.*]] = insertelement poison, i32 [[C:%.*]], i64 0 ; CHECK-NEXT: [[C_SPLAT:%.*]] = shufflevector [[C_INSERT]], poison, zeroinitializer ; CHECK-NEXT: call void @use_nxv4i1( [[A_REV]]) -; CHECK-NEXT: [[SELECT:%.*]] = select [[A_REV]], [[C_SPLAT]], [[B_REV]] +; CHECK-NEXT: [[SELECT1:%.*]] = select [[A]], [[C_SPLAT]], [[B:%.*]] +; CHECK-NEXT: [[SELECT:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[SELECT1]]) ; CHECK-NEXT: ret [[SELECT]] ; %a.rev = tail call @llvm.experimental.vector.reverse.nxv4i1( %a) @@ -569,12 +562,12 @@ ; %b.rev has multiple uses define @select_reverse_splat_true_2( %a, %b, i32 %c) { ; CHECK-LABEL: @select_reverse_splat_true_2( -; CHECK-NEXT: [[A_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i1( [[A:%.*]]) ; CHECK-NEXT: [[B_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i32( [[B:%.*]]) ; CHECK-NEXT: [[C_INSERT:%.*]] = insertelement poison, i32 [[C:%.*]], i64 0 ; CHECK-NEXT: [[C_SPLAT:%.*]] = shufflevector [[C_INSERT]], poison, zeroinitializer ; CHECK-NEXT: call void @use_nxv4i32( [[B_REV]]) -; CHECK-NEXT: [[SELECT:%.*]] = select [[A_REV]], [[C_SPLAT]], [[B_REV]] +; CHECK-NEXT: [[SELECT1:%.*]] = select [[A:%.*]], [[C_SPLAT]], [[B]] +; CHECK-NEXT: [[SELECT:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[SELECT1]]) ; CHECK-NEXT: ret [[SELECT]] ; %a.rev = tail call @llvm.experimental.vector.reverse.nxv4i1( %a) @@ -667,17 +660,13 @@ define @reverse_select_reverse( %a, %b, %c) { ; CHECK-LABEL: @reverse_select_reverse( -; CHECK-NEXT: [[A_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4i1( [[A:%.*]]) -; CHECK-NEXT: [[B_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4f32( [[B:%.*]]) -; CHECK-NEXT: [[C_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4f32( [[C:%.*]]) -; CHECK-NEXT: [[SELECT:%.*]] = select [[A_REV]], [[B_REV]], [[C_REV]] -; CHECK-NEXT: [[SELECT_REV:%.*]] = tail call @llvm.experimental.vector.reverse.nxv4f32( [[SELECT]]) -; CHECK-NEXT: ret [[SELECT_REV]] +; CHECK-NEXT: [[SELECT1:%.*]] = select fast [[A:%.*]], [[B:%.*]], [[C:%.*]] +; CHECK-NEXT: ret [[SELECT1]] ; %a.rev = tail call @llvm.experimental.vector.reverse.nxv4i1( %a) %b.rev = tail call @llvm.experimental.vector.reverse.nxv4f32( %b) %c.rev = tail call @llvm.experimental.vector.reverse.nxv4f32( %c) - %select = select %a.rev, %b.rev, %c.rev + %select = select fast %a.rev, %b.rev, %c.rev %select.rev = tail call @llvm.experimental.vector.reverse.nxv4f32( %select) ret %select.rev }