diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -25,11 +25,14 @@ #define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H #include "VPlan.h" +#include "VPlanPatternMatch.h" #include "llvm/ADT/SmallSet.h" #include "llvm/Support/InstructionCost.h" namespace llvm { +using namespace PatternMatch; + class LoopInfo; class LoopVectorizationLegality; class LoopVectorizationCostModel; @@ -149,6 +152,9 @@ VPValue *createOr(VPValue *LHS, VPValue *RHS, DebugLoc DL, const Twine &Name = "") { + if (VPValue *V = simplifyOr(LHS, RHS)) + return V; + return createInstruction(Instruction::BinaryOps::Or, {LHS, RHS}, DL, Name); } @@ -158,6 +164,21 @@ Name); } + //===--------------------------------------------------------------------===// + // VPInstruction simplifiers. + //===--------------------------------------------------------------------===// + + VPValue *simplifyOr(VPValue *LHS, VPValue *RHS) const { + VPValue *A, *B; + + // (A && B) || (A && !B) ==> A + if (match(LHS, vp_LogicalAnd(vp_Value(A), vp_Value(B))) && + match(RHS, vp_c_LogicalAnd(vp_Specific(A), vp_Not(vp_Specific(B))))) + return A; + + return nullptr; + } + //===--------------------------------------------------------------------===// // RAII helpers. //===--------------------------------------------------------------------===// diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h new file mode 100644 --- /dev/null +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -0,0 +1,123 @@ +//===- VPlanPatternMatch.h - Match on the LLVM VPlan IR -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides a simple and efficient mechanism for performing general +// tree-based pattern matches on the LLVM VPlan IR. It is heavily modelled on +// PatternMatch.h +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_PATTERN_MATH_H +#define LLVM_TRANSFORMS_VECTORIZE_VPLAN_PATTERN_MATH_H + +#include "VPlan.h" +#include "llvm/IR/PatternMatch.h" + +namespace llvm { +namespace PatternMatch { + +inline bind_ty vp_Value(VPValue *&V) { return V; } + +/// Match a specified Value*. +struct vp_specificval_ty { + const VPValue *Val; + + vp_specificval_ty(const VPValue *V) : Val(V) {} + + template bool match(ITy *V) { return V == Val; } +}; + +/// Match if we have a specific specified value. +inline vp_specificval_ty vp_Specific(const VPValue *V) { return V; }; + +struct vp_is_zero { + template bool match(ITy *V) { + if (const Value *UV = V->getUnderlyingValue()) + return isa(UV) && cast(UV)->isNullValue(); + + return false; + } +}; + +/// Match any null constant or a vector with all elements equal to 0. +/// For vectors, this includes constants with undefined elements. +inline vp_is_zero vp_Zero() { return vp_is_zero(); } + +template struct VPUnaryOp_match { + LHS L; + + VPUnaryOp_match(const LHS &L) : L(L) {} + + template bool match(T *V) { + auto *I = dyn_cast(V); + + if (!I || I->getOpcode() != Opcode) + return false; + + return L.match(I->getOperand(0)); + } +}; + +template +inline VPUnaryOp_match vp_Not(const LHS &L) { + return VPUnaryOp_match(L); +} + +template +struct VPLogicalOp_match { + LHS L; + RHS R; + + VPLogicalOp_match(const LHS &L, const RHS &R) : L(L), R(R) {} + + template bool match(T *V) { + auto *I = dyn_cast(V); + + if (!I) + return false; + + if (cast(V)->getOpcode() != Instruction::Select) + return false; + + auto *Cond = cast(V)->getOperand(0); + auto *TVal = cast(V)->getOperand(1); + auto *FVal = cast(V)->getOperand(2); + + // TODO: How best can we know the operation is on i1 types? + + if (Opcode == Instruction::And) + if (vp_Zero().match(FVal)) + return (L.match(Cond) && R.match(TVal)) || + (Commutable && L.match(TVal) && R.match(Cond)); + + return false; + } +}; + +/// Matches L && R either in the form of L & R or L ? R : false. +/// Note that the latter form is poison-blocking. +template +inline VPLogicalOp_match +vp_LogicalAnd(const LHS &L, const RHS &R) { + return VPLogicalOp_match(L, R); +} + +/// Matches L && R where L and R are arbitrary values. +inline auto vp_LogicalAnd() { return vp_LogicalAnd(m_Value(), m_Value()); } + +/// Matches L && R with LHS and RHS in either order. +template +inline VPLogicalOp_match +vp_c_LogicalAnd(const LHS &L, const RHS &R) { + return VPLogicalOp_match(L, R); +} + +} // namespace PatternMatch +} // namespace llvm + +#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_PATTERN_MATH_H