diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h --- a/llvm/include/llvm/Analysis/VectorUtils.h +++ b/llvm/include/llvm/Analysis/VectorUtils.h @@ -82,13 +82,34 @@ struct VFShape { unsigned VF; // Vectorization factor. bool IsScalable; // True if the function is a scalable function. - VFISAKind ISA; // Instruction Set Architecture. SmallVector Parameters; // List of parameter informations. // Comparison operator. bool operator==(const VFShape &Other) const { - return std::tie(VF, IsScalable, ISA, Parameters) == - std::tie(Other.VF, Other.IsScalable, Other.ISA, Other.Parameters); + return std::tie(VF, IsScalable, Parameters) == + std::tie(Other.VF, Other.IsScalable, Other.Parameters); } + /// Update the parameter in position P.ParamPos to P. + void updateParam(VFParameter P) { + assert(P.ParamPos < Parameters.size() && "Invalid parameter position."); + Parameters[P.ParamPos] = P; + assert(hasValidParameterList() && "Invalid parameter list"); + } + // Retrieve the basic vectorization shape of the function, where all + // parameters are mapped to VFParamKind::Vector with \p EC + // lanes. Specifies whether the function has a Global Predicate + // argument via \p HasGlobalPred. + static VFShape get(const CallInst &CI, ElementCount EC, bool HasGlobalPred) { + SmallVector Parameters; + for (unsigned I = 0; I < CI.arg_size(); ++I) + Parameters.push_back(VFParameter({I, VFParamKind::Vector})); + if (HasGlobalPred) + Parameters.push_back( + VFParameter({CI.arg_size(), VFParamKind::GlobalPredicate})); + + return {EC.Min, EC.Scalable, Parameters}; + } + /// Sanity check on the Parameters in the VFShape. + bool hasValidParameterList() const; }; /// Holds the VFShape for a specific scalar to vector function mapping. @@ -96,11 +117,12 @@ VFShape Shape; // Classification of the vector function. StringRef ScalarName; // Scalar Function Name. StringRef VectorName; // Vector Function Name associated to this VFInfo. + VFISAKind ISA; // Instruction Set Architecture. // Comparison operator. bool operator==(const VFInfo &Other) const { - return std::tie(Shape, ScalarName, VectorName) == - std::tie(Shape, Other.ScalarName, Other.VectorName); + return std::tie(Shape, ScalarName, VectorName, ISA) == + std::tie(Shape, Other.ScalarName, Other.VectorName, Other.ISA); } }; diff --git a/llvm/lib/Analysis/VFABIDemangling.cpp b/llvm/lib/Analysis/VFABIDemangling.cpp --- a/llvm/lib/Analysis/VFABIDemangling.cpp +++ b/llvm/lib/Analysis/VFABIDemangling.cpp @@ -402,8 +402,8 @@ assert(Parameters.back().ParamKind == VFParamKind::GlobalPredicate && "The global predicate must be the last parameter"); - const VFShape Shape({VF, IsScalable, ISA, Parameters}); - return VFInfo({Shape, ScalarName, VectorName}); + const VFShape Shape({VF, IsScalable, Parameters}); + return VFInfo({Shape, ScalarName, VectorName, ISA}); } VFParamKind VFABI::getVFParamKindFromString(const StringRef Token) { diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -1182,3 +1182,47 @@ VariantMappings.push_back(S); } } + +bool VFShape::hasValidParameterList() const { + for (unsigned Pos = 0, NumParams = Parameters.size(); Pos < NumParams; + ++Pos) { + if (Parameters[Pos].ParamPos != Pos) + return false; + switch (Parameters[Pos].ParamKind) { + default: // Nothign to check. + break; + case VFParamKind::OMP_Linear: + case VFParamKind::OMP_LinearRef: + case VFParamKind::OMP_LinearVal: + case VFParamKind::OMP_LinearUVal: + // Compile time linear steps must be non-zero. + if (Parameters[Pos].LinearStepOrPos == 0) + return false; + break; + case VFParamKind::OMP_LinearPos: + case VFParamKind::OMP_LinearRefPos: + case VFParamKind::OMP_LinearValPos: + case VFParamKind::OMP_LinearUValPos: + // The runtime linear step must be referring to some other + // parameters in the signature. + if (Parameters[Pos].LinearStepOrPos >= int(NumParams)) + return false; + // The linear step parameter must be marked as uniform. + if (Parameters[Parameters[Pos].LinearStepOrPos].ParamKind != + VFParamKind::OMP_Uniform) + return false; + // The linear step parameter can't point at itself. + if (Parameters[Pos].LinearStepOrPos == int(Pos)) + return false; + break; + case VFParamKind::GlobalPredicate: + // The global predicate must be the last one, as required by the + // Vector Function ABIs supported by LLVM (x86, AArch64, LLVM + // internal). + if (Parameters[Pos].ParamPos != NumParams - 1) + return false; + break; + } + } + return true; +} diff --git a/llvm/unittests/Analysis/VectorFunctionABITest.cpp b/llvm/unittests/Analysis/VectorFunctionABITest.cpp --- a/llvm/unittests/Analysis/VectorFunctionABITest.cpp +++ b/llvm/unittests/Analysis/VectorFunctionABITest.cpp @@ -89,7 +89,7 @@ protected: // Referencies to the parser output field. unsigned &VF = Info.Shape.VF; - VFISAKind &ISA = Info.Shape.ISA; + VFISAKind &ISA = Info.ISA; SmallVector &Parameters = Info.Shape.Parameters; StringRef &ScalarName = Info.ScalarName; StringRef &VectorName = Info.VectorName; diff --git a/llvm/unittests/Analysis/VectorUtilsTest.cpp b/llvm/unittests/Analysis/VectorUtilsTest.cpp --- a/llvm/unittests/Analysis/VectorUtilsTest.cpp +++ b/llvm/unittests/Analysis/VectorUtilsTest.cpp @@ -279,3 +279,213 @@ "}\n"); EXPECT_EQ(getSplatValue(A), nullptr); } + +//////////////////////////////////////////////////////////////////////////////// +// VFShape API tests. +//////////////////////////////////////////////////////////////////////////////// + +class VFShapeAPITest : public testing::Test { +protected: + void SetUp() override { + M = parseAssemblyString(IR, Err, Ctx); + // Get the only call instruction in the block, which is the first + // instruction. + CI = dyn_cast(&*(instructions(M->getFunction("f")).begin())); + } + const char *IR = "define i32 @f(i32 %a, i64 %b, double %c) {\n" + " %1 = call i32 @g(i32 %a, i64 %b, double %c)\n" + " ret i32 %1\n" + "}\n" + "declare i32 @g(i32, i64, double)\n"; + LLVMContext Ctx; + SMDiagnostic Err; + std::unique_ptr M; + CallInst *CI; + // Dummy shape with no parameters, overwritten by buildShape when invoked. + VFShape Shape = {/*VF*/ 2, /*IsScalable*/ false, /*Parameters*/ {}}; + VFShape Expected; + SmallVector &ExpectedParams = Expected.Parameters; + void buildShape(unsigned VF, bool IsScalable, bool HasGlobalPred) { + Shape = VFShape::get(*CI, {VF, IsScalable}, HasGlobalPred); + } + bool validParams(ArrayRef Parameters) { + Shape.Parameters = + SmallVector(Parameters.begin(), Parameters.end()); + return Shape.hasValidParameterList(); + } +}; + +TEST_F(VFShapeAPITest, API_buildVFShape) { + buildShape(/*VF*/ 2, /*IsScalable*/ false, /*HasGlobalPred*/ false); + Expected = {/*VF*/ 2, /*IsScalable*/ false, /*Parameters*/ { + {0, VFParamKind::Vector}, + {1, VFParamKind::Vector}, + {2, VFParamKind::Vector}, + }}; + EXPECT_EQ(Shape, Expected); + + buildShape(/*VF*/ 4, /*IsScalable*/ false, /*HasGlobalPred*/ true); + Expected = {/*VF*/ 4, /*IsScalable*/ false, /*Parameters*/ { + {0, VFParamKind::Vector}, + {1, VFParamKind::Vector}, + {2, VFParamKind::Vector}, + {3, VFParamKind::GlobalPredicate}, + }}; + EXPECT_EQ(Shape, Expected); + + buildShape(/*VF*/ 16, /*IsScalable*/ true, /*HasGlobalPred*/ false); + Expected = {/*VF*/ 16, /*IsScalable*/ true, /*Parameters*/ { + {0, VFParamKind::Vector}, + {1, VFParamKind::Vector}, + {2, VFParamKind::Vector}, + }}; + EXPECT_EQ(Shape, Expected); +} + +TEST_F(VFShapeAPITest, API_updateVFShape) { + + buildShape(/*VF*/ 2, /*IsScalable*/ false, /*HasGlobalPred*/ false); + Shape.updateParam({0 /*Pos*/, VFParamKind::OMP_Linear, 1, Align(4)}); + Expected = {/*VF*/ 2, /*IsScalable*/ false, /*Parameters*/ { + {0, VFParamKind::OMP_Linear, 1, Align(4)}, + {1, VFParamKind::Vector}, + {2, VFParamKind::Vector}, + }}; + EXPECT_EQ(Shape, Expected); + + // From this point on, we update only the parameters of the VFShape, + // so we update only the reference of the expected Parameters. + Shape.updateParam({1 /*Pos*/, VFParamKind::OMP_Uniform}); + ExpectedParams = { + {0, VFParamKind::OMP_Linear, 1, Align(4)}, + {1, VFParamKind::OMP_Uniform}, + {2, VFParamKind::Vector}, + }; + EXPECT_EQ(Shape, Expected); + + Shape.updateParam({2 /*Pos*/, VFParamKind::OMP_LinearRefPos, 1}); + ExpectedParams = { + {0, VFParamKind::OMP_Linear, 1, Align(4)}, + {1, VFParamKind::OMP_Uniform}, + {2, VFParamKind::OMP_LinearRefPos, 1}, + }; + EXPECT_EQ(Shape, Expected); +} + +TEST_F(VFShapeAPITest, API_updateVFShape_GlobalPredicate) { + + buildShape(/*VF*/ 2, /*IsScalable*/ true, /*HasGlobalPred*/ true); + Shape.updateParam({1 /*Pos*/, VFParamKind::OMP_Uniform}); + Expected = {/*VF*/ 2, /*IsScalable*/ true, + /*Parameters*/ {{0, VFParamKind::Vector}, + {1, VFParamKind::OMP_Uniform}, + {2, VFParamKind::Vector}, + {3, VFParamKind::GlobalPredicate}}}; + EXPECT_EQ(Shape, Expected); +} + +TEST_F(VFShapeAPITest, Parameters_Valid) { + // ParamPos in order. + EXPECT_TRUE(validParams({{0, VFParamKind::Vector}})); + EXPECT_TRUE( + validParams({{0, VFParamKind::Vector}, {1, VFParamKind::Vector}})); + EXPECT_TRUE(validParams({{0, VFParamKind::Vector}, + {1, VFParamKind::Vector}, + {2, VFParamKind::Vector}})); + + // GlocalPredicate is last parameter. + EXPECT_TRUE(validParams({{0, VFParamKind::Vector}, + {1, VFParamKind::Vector}, + {2, VFParamKind::Vector}, + {3, VFParamKind::GlobalPredicate}})); +} + +TEST_F(VFShapeAPITest, Parameters_ValidOpenMPLinear) { +// Valid linear constant step (>0). +#define __BUILD_PARAMETERS(Kind, Val) \ + { \ + { 0, Kind, Val } \ + } + EXPECT_TRUE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_Linear, 1))); + EXPECT_TRUE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearRef, 2))); + EXPECT_TRUE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearVal, 4))); + EXPECT_TRUE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearUVal, 33))); +#undef __BUILD_PARAMETERS + +// Valid linear runtime step (the step parameter is marked uniform). +#define __BUILD_PARAMETERS(Kind) \ + { \ + {0, VFParamKind::OMP_Uniform}, {1, VFParamKind::Vector}, { 2, Kind, 0 } \ + } + EXPECT_TRUE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearPos))); + EXPECT_TRUE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearRefPos))); + EXPECT_TRUE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearValPos))); + EXPECT_TRUE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearUValPos))); +#undef __BUILD_PARAMETERS +} + +TEST_F(VFShapeAPITest, Parameters_Invalid) { + // Wrong order. + EXPECT_FALSE(validParams({{1, VFParamKind::Vector}})); + EXPECT_FALSE( + validParams({{1, VFParamKind::Vector}, {0, VFParamKind::Vector}})); + + // GlobalPredicate not last parameter. + EXPECT_FALSE(validParams({{0, VFParamKind::Vector}, + {1, VFParamKind::GlobalPredicate}, + {2, VFParamKind::Vector}})); + // GlobalPredicate is not unique + EXPECT_FALSE(validParams({{0, VFParamKind::Vector}, + {1, VFParamKind::GlobalPredicate}, + {2, VFParamKind::GlobalPredicate}})); + EXPECT_FALSE(validParams({{0, VFParamKind::GlobalPredicate}, + {1, VFParamKind::Vector}, + {2, VFParamKind::GlobalPredicate}})); +} + +TEST_F(VFShapeAPITest, Parameters_InvalidOpenMPLinear) { +// Compile time linear steps must be non-zero (compile time invariant). +#define __BUILD_PARAMETERS(Kind) \ + { \ + { 0, Kind, 0 } \ + } + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_Linear))); + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearRef))); + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearVal))); + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearUVal))); +#undef __BUILD_PARAMETERS + +// The step of a runtime linear parameter must be marked +// as uniform (runtime invariant). +#define __BUILD_PARAMETERS(Kind) \ + { \ + {0, VFParamKind::OMP_Uniform}, {1, VFParamKind::Vector}, { 2, Kind, 1 } \ + } + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearPos))); + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearRefPos))); + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearValPos))); + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearUValPos))); +#undef __BUILD_PARAMETERS + +// The linear step parameter can't point at itself. +#define __BUILD_PARAMETERS(Kind) \ + { \ + {0, VFParamKind::Vector}, {1, VFParamKind::Vector}, { 2, Kind, 2 } \ + } + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearPos))); + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearRefPos))); + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearValPos))); + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearUValPos))); +#undef __BUILD_PARAMETERS + +// Linear parameter (runtime) is out of range. +#define __BUILD_PARAMETERS(Kind) \ + { \ + {0, VFParamKind::Vector}, {1, VFParamKind::Vector}, { 2, Kind, 3 } \ + } + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearPos))); + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearRefPos))); + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearValPos))); + EXPECT_FALSE(validParams(__BUILD_PARAMETERS(VFParamKind::OMP_LinearUValPos))); +#undef __BUILD_PARAMETERS +}