[mlir][NFC] Move passes in test/lib/Transforms/ to a directory that mirrors what they test

[rriddle]

Summary

test/lib/Transforms/ has bitrot and become somewhat of a dumping grounds for testing pretty much any part of the project. This revision cleans this up, and moves the files within to a directory that reflects what is actually being tested.

[herhut]

Thank you for cleaning this up!

[jpienaar]

Thanks!

mlir/test/lib/Analysis/CMakeLists.txt

 # Exclude tests from libMLIR.so
 add_mlir_library(MLIRTestAnalysis
   TestAliasAnalysis.cpp
+  TestCallGraph.cpp
+  TestLiveness.cpp
+  TestMemRefBoundCheck.cpp
+  TestMemRefDependenceCheck.cpp
+  TestMemRefStrideCalculation.cpp
+  TestNumberOfExecutions.cpp
 
-  EXCLUDE_FROM_LIBMLIR
-
-  ADDITIONAL_HEADER_DIRS
-  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Transforms
 
-  DEPENDS
-  MLIRStandardOpsIncGen
+  EXCLUDE_FROM_LIBMLIR
 
   LINK_LIBS PUBLIC
+  MLIRAffine
   MLIRAnalysis
+  MLIRMemRef
   MLIRPass
   MLIRTestDialect
   )
 
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../Dialect/Test)
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/../Dialect/Test)

mlir/test/lib/Analysis/TestCallGraph.cpp

 //===- TestCallGraph.cpp - Test callgraph construction and iteration ------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains test passes for constructing and iterating over a
 // callgraph.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Analysis/CallGraph.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 
 namespace {
 struct TestCallGraphPass
     : public PassWrapper<TestCallGraphPass, OperationPass<ModuleOp>> {
   void runOnOperation() override {
     llvm::errs() << "Testing : " << getOperation()->getAttr("test.name")
                  << "\n";
     getAnalysis<CallGraph>().print(llvm::errs());
   }
 };
 } // end anonymous namespace
 
 namespace mlir {
 namespace test {
 void registerTestCallGraphPass() {
   PassRegistration<TestCallGraphPass> pass(
       "test-print-callgraph", "Print the contents of a constructed callgraph.");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Analysis/TestLiveness.cpp

 //===- TestLiveness.cpp - Test liveness construction and information
 //-------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains test passes for constructing and resolving liveness
 // information.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Analysis/Liveness.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 
 namespace {
 
 struct TestLivenessPass : public PassWrapper<TestLivenessPass, FunctionPass> {
   void runOnFunction() override {
     llvm::errs() << "Testing : " << getFunction().getName() << "\n";
     getAnalysis<Liveness>().print(llvm::errs());
   }
 };
 
 } // end anonymous namespace
 
 namespace mlir {
 namespace test {
 void registerTestLivenessPass() {
   PassRegistration<TestLivenessPass>(
       "test-print-liveness",
       "Print the contents of a constructed liveness information.");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Analysis/TestMemRefBoundCheck.cpp

 // accesses.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Analysis/AffineAnalysis.h"
 #include "mlir/Analysis/AffineStructures.h"
 #include "mlir/Analysis/Utils.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/Pass/Pass.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "memref-bound-check"
 
 using namespace mlir;

mlir/test/lib/Analysis/TestMemRefDependenceCheck.cpp

 //===- TestMemRefDependenceCheck.cpp - Test dep analysis ------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements a pass to run pair-wise memref access dependence checks.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Analysis/AffineAnalysis.h"
 #include "mlir/Analysis/AffineStructures.h"
 #include "mlir/Analysis/Utils.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/Pass/Pass.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "test-memref-dependence-check"
 
 using namespace mlir;
 

mlir/test/lib/Analysis/TestMemRefStrideCalculation.cpp

 //===- TestMemRefStrideCalculation.cpp - Pass to test strides computation--===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/Pass/Pass.h"
-#include "mlir/Transforms/Passes.h"
 
 using namespace mlir;
 
 namespace {
 struct TestMemRefStrideCalculation
     : public PassWrapper<TestMemRefStrideCalculation, FunctionPass> {
   void runOnFunction() override;
 };

mlir/test/lib/Analysis/TestNumberOfExecutions.cpp

 //===- TestNumberOfExecutions.cpp - Test number of executions analysis ----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains test passes for constructing and resolving number of
 // executions information.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Analysis/NumberOfExecutions.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 
 namespace {
 
 struct TestNumberOfBlockExecutionsPass
     : public PassWrapper<TestNumberOfBlockExecutionsPass, FunctionPass> {
   void runOnFunction() override {
     llvm::errs() << "Number of executions: " << getFunction().getName() << "\n";
     getAnalysis<NumberOfExecutions>().printBlockExecutions(
         llvm::errs(), &getFunction().getBody());
   }
 };
 
 struct TestNumberOfOperationExecutionsPass
     : public PassWrapper<TestNumberOfOperationExecutionsPass, FunctionPass> {
   void runOnFunction() override {
     llvm::errs() << "Number of executions: " << getFunction().getName() << "\n";
     getAnalysis<NumberOfExecutions>().printOperationExecutions(
         llvm::errs(), &getFunction().getBody());
   }
 };
 
 } // end anonymous namespace
 
 namespace mlir {
 namespace test {
 void registerTestNumberOfBlockExecutionsPass() {
   PassRegistration<TestNumberOfBlockExecutionsPass>(
       "test-print-number-of-block-executions",
       "Print the contents of a constructed number of executions analysis for "
       "all blocks.");
 }
 
 void registerTestNumberOfOperationExecutionsPass() {
   PassRegistration<TestNumberOfOperationExecutionsPass>(
       "test-print-number-of-operation-executions",
       "Print the contents of a constructed number of executions analysis for "
       "all operations.");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/CMakeLists.txt

 add_subdirectory(Analysis)
+add_subdirectory(Conversion)
 add_subdirectory(Dialect)
 add_subdirectory(IR)
 add_subdirectory(Pass)
 add_subdirectory(Reducer)
 add_subdirectory(Rewrite)
 add_subdirectory(Transforms)

mlir/test/lib/Conversion/CMakeLists.txt

+add_subdirectory(StandardToLLVM)

mlir/test/lib/Conversion/StandardToLLVM/CMakeLists.txt

+# Exclude tests from libMLIR.so
+add_mlir_library(MLIRTestStandardToLLVM
+  TestConvertCallOp.cpp
+
+  EXCLUDE_FROM_LIBMLIR
+
+  LINK_LIBS PUBLIC
+  MLIRLLVMIR
+  MLIRLLVMIRTransforms
+  MLIRPass
+  MLIRStandardToLLVM
+  MLIRTestDialect
+  )
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../Dialect/Test)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/../../Dialect/Test)

mlir/test/lib/Conversion/StandardToLLVM/TestConvertCallOp.cpp

 //===- TestConvertCallOp.cpp - Test LLVM Conversion of Standard CallOp ----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "TestDialect.h"
 #include "TestTypes.h"
 #include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
 #include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 
 namespace {
 
 class TestTypeProducerOpConverter
     : public ConvertOpToLLVMPattern<test::TestTypeProducerOp> {
 public:
   using ConvertOpToLLVMPattern<
       test::TestTypeProducerOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
   matchAndRewrite(test::TestTypeProducerOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
     rewriter.replaceOpWithNewOp<LLVM::NullOp>(op, getVoidPtrType());
     return success();
   }
 };
 
 class TestConvertCallOp
     : public PassWrapper<TestConvertCallOp, OperationPass<ModuleOp>> {
 public:
   void getDependentDialects(DialectRegistry &registry) const final {
     registry.insert<LLVM::LLVMDialect>();
   }
 
   void runOnOperation() override {
     ModuleOp m = getOperation();
 
     // Populate type conversions.
     LLVMTypeConverter typeConverter(m.getContext());
     typeConverter.addConversion([&](test::TestType type) {
       return LLVM::LLVMPointerType::get(IntegerType::get(m.getContext(), 8));
     });
 
     // Populate patterns.
     RewritePatternSet patterns(m.getContext());
     populateStdToLLVMConversionPatterns(typeConverter, patterns);
     patterns.add<TestTypeProducerOpConverter>(typeConverter);
 
     // Set target.
     ConversionTarget target(getContext());
     target.addLegalDialect<LLVM::LLVMDialect>();
     target.addIllegalDialect<test::TestDialect>();
     target.addIllegalDialect<StandardOpsDialect>();
 
     if (failed(applyPartialConversion(m, target, std::move(patterns))))
       signalPassFailure();
   }
 };
 
 } // namespace
 
 namespace mlir {
 namespace test {
 void registerConvertCallOpPass() {
   PassRegistration<TestConvertCallOp>(
       "test-convert-call-op",
       "Tests conversion of `std.call` to `llvm.call` in "
       "presence of custom types");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Dialect/Affine/CMakeLists.txt

 # Exclude tests from libMLIR.so
 add_mlir_library(MLIRAffineTransformsTestPasses
   TestAffineDataCopy.cpp
   TestAffineLoopUnswitching.cpp
+  TestAffineLoopParametricTiling.cpp
   TestLoopPermutation.cpp
   TestVectorizationUtils.cpp
 
   EXCLUDE_FROM_LIBMLIR
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Affine
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/IR

mlir/test/lib/Dialect/Affine/TestAffineLoopParametricTiling.cpp

 //= TestAffineLoopParametricTiling.cpp -- Parametric Affine loop tiling pass =//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements a test pass to test parametric tiling of perfectly
 // nested affine for loops.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/Passes.h"
 #include "mlir/Transforms/LoopUtils.h"
 
 using namespace mlir;
 
 #define DEBUG_TYPE "test-affine-parametric-tile"
 
 namespace {
 struct TestAffineLoopParametricTiling
     : public PassWrapper<TestAffineLoopParametricTiling, FunctionPass> {
   void runOnFunction() override;
 };
 } // end anonymous namespace
 
 /// Checks if the function enclosing the loop nest has any arguments passed to
 /// it, which can be used as tiling parameters. Assumes that atleast 'n'
 /// arguments are passed, where 'n' is the number of loops in the loop nest.
 static void checkIfTilingParametersExist(ArrayRef<AffineForOp> band) {
   assert(!band.empty() && "no loops in input band");
   AffineForOp topLoop = band[0];
 
   if (FuncOp funcOp = dyn_cast<FuncOp>(topLoop->getParentOp()))
     assert(funcOp.getNumArguments() >= band.size() && "Too few tile sizes");
 }
 
 /// Captures tiling parameters, which are expected to be passed as arguments
 /// to the function enclosing the loop nest. Also checks if the required
 /// parameters are of index type. This approach is temporary for testing
 /// purposes.
 static void getTilingParameters(ArrayRef<AffineForOp> band,
                                 SmallVectorImpl<Value> &tilingParameters) {
   AffineForOp topLoop = band[0];
   Region *funcOpRegion = topLoop->getParentRegion();
   unsigned nestDepth = band.size();
 
   for (BlockArgument blockArgument :
        funcOpRegion->getArguments().take_front(nestDepth)) {
     if (blockArgument.getArgNumber() < nestDepth) {
       assert(blockArgument.getType().isIndex() &&
              "expected tiling parameters to be of index type.");
       tilingParameters.push_back(blockArgument);
     }
   }
 }
 
 void TestAffineLoopParametricTiling::runOnFunction() {
   // Bands of loops to tile.
   std::vector<SmallVector<AffineForOp, 6>> bands;
   getTileableBands(getFunction(), &bands);
 
   // Tile each band.
   for (SmallVectorImpl<AffineForOp> &band : bands) {
     // Capture the tiling parameters from the arguments to the function
     // enclosing this loop nest.
     SmallVector<AffineForOp, 6> tiledNest;
     SmallVector<Value, 6> tilingParameters;
     // Check if tiling parameters are present.
     checkIfTilingParametersExist(band);
 
     // Get function arguments as tiling parameters.
     getTilingParameters(band, tilingParameters);
 
     if (failed(
             tilePerfectlyNestedParametric(band, tilingParameters, &tiledNest)))
       return signalPassFailure();
   }
 }
 
 namespace mlir {
 namespace test {
 void registerTestAffineLoopParametricTilingPass() {
   PassRegistration<TestAffineLoopParametricTiling>(
       "test-affine-parametric-tile",
       "Tile affine loops using SSA values as tile sizes");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Dialect/CMakeLists.txt

 add_subdirectory(Affine)
+add_subdirectory(DLTI)
+add_subdirectory(GPU)
+add_subdirectory(Linalg)
+add_subdirectory(Math)
+add_subdirectory(SCF)
 add_subdirectory(Shape)
 add_subdirectory(SPIRV)
+add_subdirectory(StandardOps)
 add_subdirectory(Test)
 add_subdirectory(Tosa)
+add_subdirectory(Vector)

mlir/test/lib/Dialect/DLTI/CMakeLists.txt

+# Exclude tests from libMLIR.so
+add_mlir_library(MLIRDLTITestPasses
+  TestDataLayoutQuery.cpp
+
+  EXCLUDE_FROM_LIBMLIR
+
+  LINK_LIBS PUBLIC
+  MLIRDLTI
+  MLIRPass
+  MLIRTestDialect
+  )
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../Test)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/../Test)

mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp

 //===- TestDataLayoutQuery.cpp - Test Data Layout Queries -----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "TestDialect.h"
 #include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 
 namespace {
 
 /// A pass that finds "test.data_layout_query" operations and attaches to them
 /// attributes containing the results of data layout queries for operation
 /// result types.
 struct TestDataLayoutQuery
     : public PassWrapper<TestDataLayoutQuery, FunctionPass> {
   void runOnFunction() override {
     FuncOp func = getFunction();
     Builder builder(func.getContext());
     DenseMap<Operation *, DataLayout> layouts;
 
     func.walk([&](test::DataLayoutQueryOp op) {
       // Skip the ops with already processed in a deeper call.
       if (op->getAttr("size"))
         return;
 
       auto scope = op->getParentOfType<test::OpWithDataLayoutOp>();
       if (!layouts.count(scope)) {
         layouts.try_emplace(
             scope, scope ? cast<DataLayoutOpInterface>(scope.getOperation())
                          : nullptr);
       }
       auto module = op->getParentOfType<ModuleOp>();
       if (!layouts.count(module))
         layouts.try_emplace(module, module);
 
       Operation *closest = (scope && module && module->isProperAncestor(scope))
                                ? scope.getOperation()
                                : module.getOperation();
 
       const DataLayout &layout = layouts.find(closest)->getSecond();
       unsigned size = layout.getTypeSize(op.getType());
       unsigned bitsize = layout.getTypeSizeInBits(op.getType());
       unsigned alignment = layout.getTypeABIAlignment(op.getType());
       unsigned preferred = layout.getTypePreferredAlignment(op.getType());
       op->setAttrs(
           {builder.getNamedAttr("size", builder.getIndexAttr(size)),
            builder.getNamedAttr("bitsize", builder.getIndexAttr(bitsize)),
            builder.getNamedAttr("alignment", builder.getIndexAttr(alignment)),
            builder.getNamedAttr("preferred", builder.getIndexAttr(preferred))});
     });
   }
 };
 } // namespace
 
 namespace mlir {
 namespace test {
 void registerTestDataLayoutQuery() {
   PassRegistration<TestDataLayoutQuery>("test-data-layout-query",
                                         "Test data layout queries");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Dialect/GPU/CMakeLists.txt

+# Exclude tests from libMLIR.so
+add_mlir_library(MLIRGPUTestPasses
+  TestConvertGPUKernelToCubin.cpp
+  TestConvertGPUKernelToHsaco.cpp
+  TestGpuMemoryPromotion.cpp
+  TestGpuParallelLoopMapping.cpp
+  TestGpuRewrite.cpp
+
+  EXCLUDE_FROM_LIBMLIR
+
+  LINK_LIBS PUBLIC
+  MLIRAffine
+  MLIRGPU
+  MLIRGPUToGPURuntimeTransforms
+  MLIRLLVMToLLVMIRTranslation
+  MLIRNVVMIR
+  MLIRNVVMToLLVMIRTranslation
+  MLIRPass
+  MLIRROCDLIR
+  MLIRROCDLToLLVMIRTranslation
+  MLIRSCF
+  MLIRStandard
+  MLIRTransformUtils
+  )

mlir/test/lib/Dialect/GPU/TestConvertGPUKernelToCubin.cpp

 //===- TestConvertGPUKernelToCubin.cpp - Test gpu kernel cubin lowering ---===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/GPU/Passes.h"
 
 #include "mlir/Pass/Pass.h"
 #include "mlir/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Export.h"
 #include "llvm/Support/TargetSelect.h"
 
 using namespace mlir;
 
 #if MLIR_CUDA_CONVERSIONS_ENABLED
 namespace {
 class TestSerializeToCubinPass
     : public PassWrapper<TestSerializeToCubinPass, gpu::SerializeToBlobPass> {
 public:
   TestSerializeToCubinPass();
 
 private:
   void getDependentDialects(DialectRegistry &registry) const override;
 
   // Serializes PTX to CUBIN.
   std::unique_ptr<std::vector<char>>
   serializeISA(const std::string &isa) override;
 };
 } // namespace
 
 TestSerializeToCubinPass::TestSerializeToCubinPass() {
   this->triple = "nvptx64-nvidia-cuda";
   this->chip = "sm_35";
   this->features = "+ptx60";
 }
 
 void TestSerializeToCubinPass::getDependentDialects(
     DialectRegistry &registry) const {
   registerNVVMDialectTranslation(registry);
   gpu::SerializeToBlobPass::getDependentDialects(registry);
 }
 
 std::unique_ptr<std::vector<char>>
 TestSerializeToCubinPass::serializeISA(const std::string &) {
   std::string data = "CUBIN";
   return std::make_unique<std::vector<char>>(data.begin(), data.end());
 }
 
 namespace mlir {
 namespace test {
 // Register test pass to serialize GPU module to a CUBIN binary annotation.
 void registerTestGpuSerializeToCubinPass() {
   PassRegistration<TestSerializeToCubinPass> registerSerializeToCubin(
       "test-gpu-to-cubin",
       "Lower GPU kernel function to CUBIN binary annotations", [] {
         // Initialize LLVM NVPTX backend.
         LLVMInitializeNVPTXTarget();
         LLVMInitializeNVPTXTargetInfo();
         LLVMInitializeNVPTXTargetMC();
         LLVMInitializeNVPTXAsmPrinter();
 
         return std::make_unique<TestSerializeToCubinPass>();
       });
 }
 } // namespace test
 } // namespace mlir
 #endif // MLIR_CUDA_CONVERSIONS_ENABLED

mlir/test/lib/Dialect/GPU/TestConvertGPUKernelToHsaco.cpp

 //===- TestConvertGPUKernelToHsaco.cpp - Test gpu kernel hsaco lowering ---===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/GPU/Passes.h"
 
 #include "mlir/Pass/Pass.h"
 #include "mlir/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Export.h"
 #include "llvm/Support/TargetSelect.h"
 
 using namespace mlir;
 
 #if MLIR_ROCM_CONVERSIONS_ENABLED
 namespace {
 class TestSerializeToHsacoPass
     : public PassWrapper<TestSerializeToHsacoPass, gpu::SerializeToBlobPass> {
 public:
   TestSerializeToHsacoPass();
 
 private:
   void getDependentDialects(DialectRegistry &registry) const override;
 
   // Serializes ROCDL IR to HSACO.
   std::unique_ptr<std::vector<char>>
   serializeISA(const std::string &isa) override;
 };
 } // namespace
 
 TestSerializeToHsacoPass::TestSerializeToHsacoPass() {
   this->triple = "amdgcn-amd-amdhsa";
   this->chip = "gfx900";
 }
 
 void TestSerializeToHsacoPass::getDependentDialects(
     DialectRegistry &registry) const {
   registerROCDLDialectTranslation(registry);
   gpu::SerializeToBlobPass::getDependentDialects(registry);
 }
 
 std::unique_ptr<std::vector<char>>
 TestSerializeToHsacoPass::serializeISA(const std::string &) {
   std::string data = "HSACO";
   return std::make_unique<std::vector<char>>(data.begin(), data.end());
 }
 
 namespace mlir {
 namespace test {
 // Register test pass to serialize GPU module to a HSAco binary annotation.
 void registerTestGpuSerializeToHsacoPass() {
   PassRegistration<TestSerializeToHsacoPass> registerSerializeToHsaco(
       "test-gpu-to-hsaco",
       "Lower GPU kernel function to HSAco binary annotations", [] {
         // Initialize LLVM AMDGPU backend.
         LLVMInitializeAMDGPUTarget();
         LLVMInitializeAMDGPUTargetInfo();
         LLVMInitializeAMDGPUTargetMC();
         LLVMInitializeAMDGPUAsmPrinter();
 
         return std::make_unique<TestSerializeToHsacoPass>();
       });
 }
 } // namespace test
 } // namespace mlir
 #endif // MLIR_ROCM_CONVERSIONS_ENABLED

mlir/test/lib/Dialect/GPU/TestGpuMemoryPromotion.cpp

 //===- TestGPUMemoryPromotionPass.cpp - Test pass for GPU promotion -------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the pass testing the utilities for moving data across
 // different levels of the GPU memory hierarchy.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/GPU/GPUDialect.h"
 #include "mlir/Dialect/GPU/MemoryPromotion.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/SCF.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 
 namespace {
 /// Simple pass for testing the promotion to workgroup memory in GPU functions.
 /// Promotes all arguments with "gpu.test_promote_workgroup" attribute. This
 /// does not check whether the promotion is legal (e.g., amount of memory used)
 /// or beneficial (e.g., makes previously uncoalesced loads coalesced).
 class TestGpuMemoryPromotionPass
     : public PassWrapper<TestGpuMemoryPromotionPass,
                          OperationPass<gpu::GPUFuncOp>> {
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<AffineDialect, memref::MemRefDialect, StandardOpsDialect,
                     scf::SCFDialect>();
   }
 
   void runOnOperation() override {
     gpu::GPUFuncOp op = getOperation();
     for (unsigned i = 0, e = op.getNumArguments(); i < e; ++i) {
       if (op.getArgAttrOfType<UnitAttr>(i, "gpu.test_promote_workgroup"))
         promoteToWorkgroupMemory(op, i);
     }
   }
 };
 } // end namespace
 
 namespace mlir {
 void registerTestGpuMemoryPromotionPass() {
   PassRegistration<TestGpuMemoryPromotionPass>(
       "test-gpu-memory-promotion",
       "Promotes the annotated arguments of gpu.func to workgroup memory.");
 }
 } // namespace mlir

mlir/test/lib/Dialect/GPU/TestGpuParallelLoopMapping.cpp

 //===- TestGPUParallelLoopMapping.cpp - Test pass for GPU loop mapping ----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the pass testing the utilities for mapping parallel
 // loops to gpu hardware ids.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/GPU/ParallelLoopMapper.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 
 namespace {
 /// Simple pass for testing the mapping of parallel loops to hardware ids using
 /// a greedy mapping strategy.
 class TestGpuGreedyParallelLoopMappingPass
     : public PassWrapper<TestGpuGreedyParallelLoopMappingPass,
                          OperationPass<FuncOp>> {
   void runOnOperation() override {
     Operation *op = getOperation();
     for (Region &region : op->getRegions())
       greedilyMapParallelSCFToGPU(region);
   }
 };
 } // namespace
 
 namespace mlir {
 namespace test {
 void registerTestGpuParallelLoopMappingPass() {
   PassRegistration<TestGpuGreedyParallelLoopMappingPass> registration(
       "test-gpu-greedy-parallel-loop-mapping",
       "Greedily maps all parallel loops to gpu hardware ids.");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Dialect/GPU/TestGpuRewrite.cpp

 //===- TestAllReduceLowering.cpp - Test gpu.all_reduce lowering -----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains test passes for lowering the gpu.all_reduce op.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/GPU/Passes.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
 
 namespace {
 struct TestGpuRewritePass
     : public PassWrapper<TestGpuRewritePass, OperationPass<ModuleOp>> {
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<StandardOpsDialect, memref::MemRefDialect>();
   }
   void runOnOperation() override {
     RewritePatternSet patterns(&getContext());
     populateGpuRewritePatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
   }
 };
 } // namespace
 
 namespace mlir {
 void registerTestAllReduceLoweringPass() {
   PassRegistration<TestGpuRewritePass> pass(
       "test-gpu-rewrite",
       "Applies all rewrite patterns within the GPU dialect.");
 }
 } // namespace mlir

mlir/test/lib/Dialect/Linalg/CMakeLists.txt

+# Exclude tests from libMLIR.so
+add_mlir_library(MLIRLinalgTestPasses
+  TestConvVectorization.cpp
+  TestLinalgCodegenStrategy.cpp
+  TestLinalgElementwiseFusion.cpp
+  TestLinalgFusionTransforms.cpp
+  TestLinalgHoisting.cpp
+  TestLinalgTransforms.cpp
+
+  EXCLUDE_FROM_LIBMLIR
+
+  LINK_LIBS PUBLIC
+  MLIRAffine
+  MLIRGPU
+  MLIRLinalg
+  MLIRLinalgTransforms
+  MLIRLLVMToLLVMIRTranslation
+  MLIRPass
+  MLIRStandard
+  MLIRTransformUtils
+  MLIRVector
+  MLIRVectorToSCF
+  )

mlir/test/lib/Dialect/Linalg/TestConvVectorization.cpp

 //===- TestConvVectorization.cpp - Vectorization of Conv ops --------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
 #include "mlir/Dialect/Linalg/Passes.h"
 #include "mlir/Dialect/Linalg/Transforms/Hoisting.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Vector/VectorTransforms.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "mlir/Transforms/LoopUtils.h"
 #include "mlir/Transforms/Passes.h"
 
 using namespace mlir;
 using namespace vector;
 
 namespace {
 /// A pass converting MLIR Linalg ops into Vector ops.
 class TestConvVectorization
     : public PassWrapper<TestConvVectorization, OperationPass<ModuleOp>> {
 public:
   TestConvVectorization() = default;
   TestConvVectorization(const TestConvVectorization &) {}
   explicit TestConvVectorization(ArrayRef<int64_t> tileSizesParam) {
     tileSizes = tileSizesParam;
   }
 
   void runOnOperation() override;
 
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<VectorDialect>();
     registry.insert<linalg::LinalgDialect>();
     registry.insert<memref::MemRefDialect>();
     registry.insert<scf::SCFDialect>();
     registry.insert<AffineDialect>();
     registry.insert<StandardOpsDialect>();
   }
 
   ListOption<int64_t> tileSizes{
       *this, "tile-sizes", llvm::cl::desc("Vectorization sizes."),
       llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated};
 };
 } // namespace
 
 void TestConvVectorization::runOnOperation() {
   MLIRContext *context = &getContext();
   ModuleOp module = getOperation();
 
   ConversionTarget target(*context);
   target.addLegalDialect<AffineDialect, scf::SCFDialect, StandardOpsDialect,
                          VectorDialect>();
   target.addLegalOp<ModuleOp, FuncOp, ReturnOp>();
   target.addLegalOp<linalg::FillOp, linalg::YieldOp>();
 
   SmallVector<RewritePatternSet, 4> stage1Patterns;
   linalg::populateConvVectorizationPatterns(context, stage1Patterns, tileSizes);
   SmallVector<FrozenRewritePatternSet, 4> frozenStage1Patterns;
   llvm::move(stage1Patterns, std::back_inserter(frozenStage1Patterns));
 
   RewritePatternSet stage2Patterns =
       linalg::getLinalgTilingCanonicalizationPatterns(context);
   stage2Patterns.add<linalg::AffineMinSCFCanonicalizationPattern>(context);
 
   auto stage3Transforms = [](Operation *op) {
     PassManager pm(op->getContext());
     pm.addPass(createLoopInvariantCodeMotionPass());
     if (failed(pm.run(cast<ModuleOp>(op))))
       llvm_unreachable("Unexpected failure in cleanup pass pipeline.");
     op->walk([](FuncOp func) {
       promoteSingleIterationLoops(func);
       linalg::hoistRedundantVectorTransfers(func);
     });
     return success();
   };
 
   (void)linalg::applyStagedPatterns(module, frozenStage1Patterns,
                                     std::move(stage2Patterns),
                                     stage3Transforms);
 
   //===--------------------------------------------------------------------===//
   // Post staged patterns transforms
   //===--------------------------------------------------------------------===//
 
   VectorTransformsOptions vectorTransformsOptions{
       VectorContractLowering::Dot, VectorTransposeLowering::EltWise};
 
   RewritePatternSet vectorTransferPatterns(context);
   // Pattern is not applied because rank-reducing vector transfer is not yet
   // supported as can be seen in splitFullAndPartialTransferPrecondition,
   // VectorTransforms.cpp
   vectorTransferPatterns.add<VectorTransferFullPartialRewriter>(
       context, vectorTransformsOptions);
   (void)applyPatternsAndFoldGreedily(module, std::move(vectorTransferPatterns));
 
   // Programmatic controlled lowering of linalg.copy and linalg.fill.
   PassManager pm(context);
   pm.addNestedPass<FuncOp>(createConvertLinalgToLoopsPass());
   if (failed(pm.run(module)))
     llvm_unreachable("Unexpected failure in linalg to loops pass.");
 
   // Programmatic controlled lowering of vector.contract only.
   RewritePatternSet vectorContractLoweringPatterns(context);
   populateVectorContractLoweringPatterns(vectorContractLoweringPatterns,
                                          vectorTransformsOptions);
   populateVectorTransposeLoweringPatterns(vectorContractLoweringPatterns,
                                           vectorTransformsOptions);
   (void)applyPatternsAndFoldGreedily(module,
                                      std::move(vectorContractLoweringPatterns));
 
   // Programmatic controlled lowering of vector.transfer only.
   RewritePatternSet vectorToLoopsPatterns(context);
   populateVectorToSCFConversionPatterns(vectorToLoopsPatterns,
                                         VectorTransferToSCFOptions());
   (void)applyPatternsAndFoldGreedily(module, std::move(vectorToLoopsPatterns));
 
   // Ensure we drop the marker in the end.
   module.walk([](linalg::LinalgOp op) {
     op->removeAttr(linalg::LinalgTransforms::kLinalgTransformMarker);
   });
 }
 
 namespace mlir {
 namespace test {
 void registerTestConvVectorization() {
   PassRegistration<TestConvVectorization> testTransformPatternsPass(
       "test-conv-vectorization", "Test vectorization of convolutions");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Dialect/Linalg/TestLinalgCodegenStrategy.cpp

 //===- TestLinalgCodegenStrategy.cpp - Test Linalg codegen strategy -------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements logic for testing the Linalg codegen strategy.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/GPU/GPUDialect.h"
 #include "mlir/Dialect/Linalg/IR/LinalgOps.h"
 #include "mlir/Dialect/Linalg/Transforms/CodegenStrategy.h"
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/Pass.h"
 
 #include "llvm/ADT/SetVector.h"
 
 using namespace mlir;
 using namespace mlir::linalg;
 
 namespace {
 struct TestLinalgCodegenStrategy
     : public PassWrapper<TestLinalgCodegenStrategy, FunctionPass> {
   TestLinalgCodegenStrategy() = default;
   TestLinalgCodegenStrategy(const TestLinalgCodegenStrategy &pass) {}
 
   void getDependentDialects(DialectRegistry &registry) const override {
     // clang-format off
     registry.insert<AffineDialect,
                     gpu::GPUDialect,
                     linalg::LinalgDialect,
                     memref::MemRefDialect,
                     scf::SCFDialect,
                     StandardOpsDialect,
                     vector::VectorDialect>();
     // clang-format on
   }
 
   template <typename LinalgNamedOp>
   void applyStrategyToNamedLinalgOp();
 
   void runOnFunction() override;
 
   template <typename OpType>
   void runStrategy(LinalgTilingOptions tilingOptions,
                    LinalgTilingOptions registerTilingOptions,
                    vector::VectorContractLowering vectorContractLowering,
                    vector::VectorTransferSplit vectorTransferSplit);
 
   ListOption<int64_t> tileSizes{*this, "tile-sizes",
                                 llvm::cl::MiscFlags::CommaSeparated,
                                 llvm::cl::desc("Specifies the tile sizes.")};
   ListOption<unsigned> tileInterchange{
       *this, "tile-interchange", llvm::cl::MiscFlags::CommaSeparated,
       llvm::cl::desc("Specifies the tile interchange.")};
 
   Option<bool> promote{
       *this, "promote",
       llvm::cl::desc("Promote the tile into a small aligned memory buffer."),
       llvm::cl::init(false)};
   Option<bool> promoteFullTile{
       *this, "promote-full-tile-pad",
       llvm::cl::desc("Pad the small aligned memory buffer to the tile sizes."),
       llvm::cl::init(false)};
   ListOption<int64_t> registerTileSizes{
       *this, "register-tile-sizes", llvm::cl::MiscFlags::CommaSeparated,
       llvm::cl::desc(
           "Specifies the size of the register tile that will be used "
           " to vectorize")};
   Option<bool> registerPromote{
       *this, "register-promote",
       llvm::cl::desc(
           "Promote the register tile into a small aligned memory buffer."),
       llvm::cl::init(false)};
   Option<bool> registerPromoteFullTile{
       *this, "register-promote-full-tile-pad",
       llvm::cl::desc("Pad the small aligned memory buffer to the tile sizes."),
       llvm::cl::init(false)};
   Option<bool> vectorize{
       *this, "vectorize",
       llvm::cl::desc("Rewrite the linalg op as a vector operation."),
       llvm::cl::init(false)};
   Option<std::string> splitVectorTransfersTo{
       *this, "split-transfers",
       llvm::cl::desc(
           "Split vector transfers between slow (masked) and fast "
           "(unmasked) variants. Possible options are:\n"
           "\tnone: keep unsplit vector.transfer and pay the full price\n"
           "\tlinalg-copy: use linalg.fill + linalg.copy for the slow path\n"
           "\tvector-transfers: use extra small unmasked vector.transfer for"
           " the slow path\n"),
       llvm::cl::init("none")};
   Option<std::string> vectorizeContractionTo{
       *this, "vectorize-contraction-to",
       llvm::cl::desc("the type of vector op to use for linalg contractions"),
       llvm::cl::init("outerproduct")};
   Option<bool> unrollVectorTransfers{
       *this, "unroll-vector-transfers",
       llvm::cl::desc("Enable full unrolling of vector.transfer operations"),
       llvm::cl::init(false)};
   Option<std::string> anchorOpName{
       *this, "anchor-op",
       llvm::cl::desc(
           "Which single linalg op is the anchor for the codegen strategy to "
           "latch on:\n"
           "\tlinalg.matmul: anchor on linalg.matmul\n"
           "\tlinalg.matmul_column_major: anchor on linalg.matmul_column_major\n"
           "\tlinalg.copy: anchor on linalg.copy\n"
           "\tlinalg.fill: anchor on linalg.fill\n"),
       llvm::cl::init("")};
   Option<std::string> anchorFuncOpName{
       *this, "anchor-func",
       llvm::cl::desc(
           "Which single func op is the anchor for the codegen strategy to "
           "latch on."),
       llvm::cl::init("")};
 };
 
 template <>
 void TestLinalgCodegenStrategy::runStrategy<LinalgOp>(
     LinalgTilingOptions tilingOptions,
     LinalgTilingOptions registerTilingOptions,
     vector::VectorContractLowering vectorContractLowering,
     vector::VectorTransferSplit vectorTransferSplit) {
   assert(!anchorOpName.empty());
   CodegenStrategy strategy;
   strategy.tileIf<LinalgOp>(!tileSizes.empty(), anchorOpName, tilingOptions)
       .promoteIf<LinalgOp>(promote, anchorOpName,
                            LinalgPromotionOptions()
                                .setAlignment(16)
                                .setUseFullTileBuffersByDefault(promoteFullTile))
       .tileIf<LinalgOp>(!registerTileSizes.empty(), anchorOpName,
                         registerTilingOptions)
       .promoteIf<LinalgOp>(
           registerPromote, anchorOpName,
           LinalgPromotionOptions()
               .setAlignment(16)
               .setUseFullTileBuffersByDefault(registerPromoteFullTile))
       .vectorizeIf(vectorize, anchorOpName)
       .setVectorTransformsOptions(
           vector::VectorTransformsOptions()
               .setVectorTransformsOptions(vectorContractLowering)
               .setVectorTransferSplit(vectorTransferSplit))
       .setVectorTransferToSCFOptions(
           VectorTransferToSCFOptions().setUnroll(unrollVectorTransfers));
   strategy.transform(getFunction());
 }
 
 template <typename OpType>
 void TestLinalgCodegenStrategy::runStrategy(
     LinalgTilingOptions tilingOptions,
     LinalgTilingOptions registerTilingOptions,
     vector::VectorContractLowering vectorContractLowering,
     vector::VectorTransferSplit vectorTransferSplit) {
   CodegenStrategy strategy;
   strategy.tileIf<OpType>(!tileSizes.empty(), tilingOptions)
       .template promoteIf<OpType>(
           promote, LinalgPromotionOptions()
                        .setAlignment(16)
                        .setUseFullTileBuffersByDefault(promoteFullTile))
       .template tileIf<OpType>(!registerTileSizes.empty(),
                                registerTilingOptions)
       .template promoteIf<OpType>(
           registerPromote,
           LinalgPromotionOptions()
               .setAlignment(16)
               .setUseFullTileBuffersByDefault(registerPromoteFullTile))
       .template vectorizeIf<OpType>(vectorize)
       .setVectorTransformsOptions(
           vector::VectorTransformsOptions()
               .setVectorTransformsOptions(vectorContractLowering)
               .setVectorTransferSplit(vectorTransferSplit))
       .setVectorTransferToSCFOptions(
           VectorTransferToSCFOptions().setUnroll(unrollVectorTransfers));
   strategy.transform(getFunction());
 }
 } // end anonymous namespace
 
 /// Apply transformations specified as patterns.
 void TestLinalgCodegenStrategy::runOnFunction() {
   if (!anchorFuncOpName.empty() && anchorFuncOpName != getFunction().getName())
     return;
 
   LinalgTilingOptions tilingOptions;
   if (!tileSizes.empty())
     tilingOptions = tilingOptions.setTileSizes(tileSizes);
   if (!tileInterchange.empty())
     tilingOptions = tilingOptions.setInterchange(tileInterchange);
 
   LinalgTilingOptions registerTilingOptions;
   if (!registerTileSizes.empty())
     registerTilingOptions =
         registerTilingOptions.setTileSizes(registerTileSizes);
 
   vector::VectorContractLowering vectorContractLowering =
       llvm::StringSwitch<vector::VectorContractLowering>(
           vectorizeContractionTo.getValue())
           .Case("matrixintrinsics", vector::VectorContractLowering::Matmul)
           .Case("dot", vector::VectorContractLowering::Dot)
           .Case("outerproduct", vector::VectorContractLowering::OuterProduct)
           .Default(vector::VectorContractLowering::OuterProduct);
   vector::VectorTransferSplit vectorTransferSplit =
       llvm::StringSwitch<vector::VectorTransferSplit>(
           splitVectorTransfersTo.getValue())
           .Case("none", vector::VectorTransferSplit::None)
           .Case("linalg-copy", vector::VectorTransferSplit::LinalgCopy)
           .Case("vector-transfers", vector::VectorTransferSplit::VectorTransfer)
           .Default(vector::VectorTransferSplit::None);
 
   // If no anchorOpNameis specified, just test that strategy applies properly to
   // linalg::MatmulOp.
   if (anchorOpName.empty())
     runStrategy<linalg::MatmulOp>(tilingOptions, registerTilingOptions,
                                   vectorContractLowering, vectorTransferSplit);
   else
     runStrategy<LinalgOp>(tilingOptions, registerTilingOptions,
                           vectorContractLowering, vectorTransferSplit);
 }
 
 namespace mlir {
 namespace test {
 void registerTestLinalgCodegenStrategy() {
   PassRegistration<TestLinalgCodegenStrategy> testLinalgCodegenStrategyPass(
       "test-linalg-codegen-strategy", "Test Linalg Codegen Strategy.");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Dialect/Linalg/TestLinalgElementwiseFusion.cpp

 //===- TestLinalgElementwiseFusion.cpp - Test Linalg elementwise fusion ---===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements a pass for testing fusion of elementwise operations in
 // Linalg, mainly linalg options.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 namespace mlir {
 
 static void addOperands(Operation *op, SetVector<Value> &operandSet) {
   if (!op)
     return;
   TypeSwitch<Operation *, void>(op)
       .Case<linalg::LinalgOp>([&](linalg::LinalgOp linalgOp) {
         operandSet.insert(linalgOp.getInputs().begin(),
                           linalgOp.getInputs().end());
       })
       .Default([&](Operation *operation) {
         operandSet.insert(operation->operand_begin(), operation->operand_end());
       });
 }
 
 template <int limit = 3>
 static bool setFusedOpOperandLimit(const OpResult &producer,
                                    const OpOperand &consumer) {
   SetVector<Value> fusedOpOperands;
   if (producer.getOwner()->getNumResults() != 1)
     return false;
   addOperands(consumer.getOwner(), fusedOpOperands);
   fusedOpOperands.remove(producer);
   addOperands(producer.getOwner(), fusedOpOperands);
   return fusedOpOperands.size() <= limit;
 }
 
 namespace {
 struct TestLinalgElementwiseFusion
     : public PassWrapper<TestLinalgElementwiseFusion, FunctionPass> {
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<AffineDialect, linalg::LinalgDialect, memref::MemRefDialect,
                     tensor::TensorDialect>();
   }
 
   void runOnFunction() override {
     MLIRContext *context = &this->getContext();
     FuncOp funcOp = this->getFunction();
     RewritePatternSet fusionPatterns(context);
 
     linalg::populateElementwiseOpsFusionPatterns(
         fusionPatterns,
         linalg::LinalgElementwiseFusionOptions()
             .setControlElementwiseOpsFusionFn(setFusedOpOperandLimit<4>));
 
     (void)applyPatternsAndFoldGreedily(funcOp.getBody(),
                                        std::move(fusionPatterns));
   }
 };
 
 struct TestPushExpandingReshape
     : public PassWrapper<TestPushExpandingReshape, FunctionPass> {
   void getDependentDialects(DialectRegistry &registry) const override {
     registry
         .insert<AffineDialect, linalg::LinalgDialect, tensor::TensorDialect>();
   }
 
   void runOnFunction() override {
     MLIRContext *context = &this->getContext();
     FuncOp funcOp = this->getFunction();
     RewritePatternSet patterns(context);
     linalg::populatePushReshapeOpsPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(funcOp.getBody(), std::move(patterns));
   }
 };
 } // namespace
 
 namespace test {
 void registerTestLinalgElementwiseFusion() {
   PassRegistration<TestLinalgElementwiseFusion> testElementwiseFusionPass(
       "test-linalg-elementwise-fusion-patterns",
       "Test Linalg element wise operation fusion patterns");
 }
 
 void registerTestPushExpandingReshape() {
   PassRegistration<TestPushExpandingReshape> testPushExpandingReshapePass(
       "test-linalg-push-reshape", "Test Linalg reshape push patterns");
 }
 } // namespace test
 
 } // namespace mlir

mlir/test/lib/Dialect/Linalg/TestLinalgFusionTransforms.cpp

 //===- TestLinalgFusionTransforms.cpp - Test Linalg fusion patterns -------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements logic for testing Linalg fusion patterns.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Linalg/Analysis/DependenceAnalysis.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "mlir/Transforms/Passes.h"
 
 using namespace mlir;
 using namespace mlir::linalg;
 
 template <LinalgTilingLoopType LoopType>
 static void fillFusionPatterns(MLIRContext *context,
                                const LinalgDependenceGraph &dependenceGraph,
                                RewritePatternSet &patterns) {
   patterns.add<LinalgTileAndFusePattern<MatmulOp>,
                LinalgTileAndFusePattern<ConvOp>>(
       context, dependenceGraph,
       LinalgTilingOptions().setTileSizes({32, 64, 16}).setLoopType(LoopType),
       LinalgFusionOptions().setIndicesToFuse({2}),
       LinalgTransformationFilter(
           Identifier::get("basic_fusion", context),
           Identifier::get("after_basic_fusion", context)),
       LinalgTransformationFilter(
           ArrayRef<Identifier>(),
           Identifier::get("after_basic_fusion_producer", context)),
       LinalgTransformationFilter(
           ArrayRef<Identifier>(),
           Identifier::get("after_basic_fusion_original", context)));
 
   patterns.add<LinalgTileAndFusePattern<MatmulOp>>(
       context, dependenceGraph,
       LinalgTilingOptions().setTileSizes({32, 64, 16}).setLoopType(LoopType),
       LinalgFusionOptions().setIndicesToFuse({0}),
       LinalgTransformationFilter(Identifier::get("lhs_fusion", context),
                                  Identifier::get("after_lhs_fusion", context)),
       LinalgTransformationFilter(
           ArrayRef<Identifier>(),
           Identifier::get("after_lhs_fusion_producer", context)),
       LinalgTransformationFilter(
           ArrayRef<Identifier>(),
           Identifier::get("after_lhs_fusion_original", context)));
 
   patterns.add<LinalgTileAndFusePattern<MatmulOp>>(
       context, dependenceGraph,
       LinalgTilingOptions().setTileSizes({32, 64, 16}).setLoopType(LoopType),
       LinalgFusionOptions().setIndicesToFuse({2}),
       LinalgTransformationFilter(Identifier::get("out_fusion", context),
                                  Identifier::get("after_out_fusion", context)),
       LinalgTransformationFilter(
           ArrayRef<Identifier>(),
           Identifier::get("after_out_fusion_producer", context)),
       LinalgTransformationFilter(
           ArrayRef<Identifier>(),
           Identifier::get("after_out_fusion_original", context)));
 
   patterns.add<LinalgTileAndFusePattern<MatmulOp>>(
       context, dependenceGraph,
       LinalgTilingOptions().setTileSizes({32, 64, 16}).setLoopType(LoopType),
       LinalgFusionOptions().setIndicesToFuse({1}),
       LinalgTransformationFilter(Identifier::get("rhs_fusion", context),
                                  Identifier::get("after_rhs_fusion", context)),
       LinalgTransformationFilter(
           ArrayRef<Identifier>(),
           Identifier::get("after_rhs_fusion_producer", context)),
       LinalgTransformationFilter(
           ArrayRef<Identifier>(),
           Identifier::get("after_rhs_fusion_original", context)));
 
   patterns.add<LinalgTileAndFusePattern<MatmulOp>>(
       context, dependenceGraph,
       LinalgTilingOptions().setTileSizes({32, 64, 16}).setLoopType(LoopType),
       LinalgFusionOptions().setIndicesToFuse({0, 2}),
       LinalgTransformationFilter(
           Identifier::get("two_operand_fusion", context),
           Identifier::get("after_two_operand_fusion", context)),
       LinalgTransformationFilter(
           ArrayRef<Identifier>(),
           Identifier::get("after_two_operand_fusion_producer", context)),
       LinalgTransformationFilter(
           ArrayRef<Identifier>(),
           Identifier::get("after_two_operand_fusion_original", context)));
 
   patterns.add<LinalgTileAndFusePattern<GenericOp>>(
       context, dependenceGraph,
       LinalgTilingOptions().setTileSizes({32, 64}).setLoopType(LoopType),
       LinalgFusionOptions().setIndicesToFuse({0, 1}),
       LinalgTransformationFilter(
           Identifier::get("transpose_fusion", context),
           Identifier::get("after_transpose_fusion", context)),
       LinalgTransformationFilter(
           ArrayRef<Identifier>(),
           Identifier::get("after_transpose_fusion_producer", context)),
       LinalgTransformationFilter(
           ArrayRef<Identifier>(),
           Identifier::get("after_transpose_fusion_original", context)));
 }
 
 namespace {
 template <LinalgTilingLoopType LoopType = LinalgTilingLoopType::ParallelLoops>
 struct TestLinalgFusionTransforms
     : public PassWrapper<TestLinalgFusionTransforms<LoopType>, FunctionPass> {
   TestLinalgFusionTransforms() = default;
   TestLinalgFusionTransforms(const TestLinalgFusionTransforms &pass) {}
 
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<AffineDialect, linalg::LinalgDialect, memref::MemRefDialect,
                     scf::SCFDialect, StandardOpsDialect>();
   }
 
   void runOnFunction() override {
     MLIRContext *context = &this->getContext();
     FuncOp funcOp = this->getFunction();
     RewritePatternSet fusionPatterns(context);
     Aliases alias;
     LinalgDependenceGraph dependenceGraph =
         LinalgDependenceGraph::buildDependenceGraph(alias, funcOp);
     fillFusionPatterns<LoopType>(context, dependenceGraph, fusionPatterns);
     (void)applyPatternsAndFoldGreedily(funcOp, std::move(fusionPatterns));
   }
 };
 } // namespace
 
 static LogicalResult fuseLinalgOpsGreedily(FuncOp f) {
   OpBuilder b(f);
   DenseSet<Operation *> eraseSet;
 
   // Save original Linalg ops, we only want to make a pass over those.
   SmallVector<LinalgOp, 8> linalgOps;
   f.walk([&](LinalgOp op) {
     // TODO: support multi-results.
     if (op->getNumResults() <= 1)
       linalgOps.push_back(op);
   });
 
   // Tile and Fuse for tensors inputs (TODO: all tensor operands).
   bool changed = false;
   for (LinalgOp linalgOp : llvm::reverse(linalgOps)) {
     for (OpOperand &opOperand : linalgOp.getShapedOpOperands()) {
       if (opOperand.get().getType().isa<MemRefType>()) {
         // TODO: LinalgDependenceGraph should be able to update itself.
         // The current naive and expensive reconstruction of the graph should be
         // removed.
         linalg::Aliases aliases;
         linalg::LinalgDependenceGraph graph(aliases, linalgOps);
         if (auto info = fuseProducerOfBuffer(b, opOperand, graph)) {
           auto *originalOp = info->originalProducer.getOperation();
           eraseSet.insert(originalOp);
           auto *originalOpInLinalgOpsVector =
               std::find(linalgOps.begin(), linalgOps.end(), originalOp);
           *originalOpInLinalgOpsVector = info->fusedProducer.getOperation();
           changed = true;
         }
       } else {
         assert(opOperand.get().getType().isa<RankedTensorType>());
         // Tile and Fuse tensor input.
         if (opOperand.getOperandNumber() >= linalgOp.getNumInputs())
           continue;
         if (auto info = fuseProducerOfTensor(b, opOperand)) {
           auto *originalOp = info->originalProducer.getOperation();
           auto *originalOpInLinalgOpsVector =
               std::find(linalgOps.begin(), linalgOps.end(), originalOp);
           *originalOpInLinalgOpsVector = info->fusedProducer.getOperation();
           // Don't mark for erasure in the tensor case, let DCE handle this.
           changed = true;
         }
       }
     }
   }
   // The `fuseProducerOfBuffer` function performs structural checks and in
   // particular that no covering read or write exist between the consumer and
   // the producer. As a consequence, the only fusions that may occur preserve
   // subsequent dependences and are guaranteed by construction to produce the
   // whole view. We may thus erase the producer once it is fused.
   for (auto *e : eraseSet)
     e->erase();
 
   return changed ? success() : failure();
 }
 
 namespace {
 struct TestLinalgGreedyFusion
     : public PassWrapper<TestLinalgGreedyFusion, FunctionPass> {
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<AffineDialect, linalg::LinalgDialect, memref::MemRefDialect,
                     scf::SCFDialect>();
   }
   void runOnFunction() override {
     MLIRContext *context = &getContext();
     RewritePatternSet patterns =
         linalg::getLinalgTilingCanonicalizationPatterns(context);
     patterns.add<AffineMinSCFCanonicalizationPattern>(context);
     FrozenRewritePatternSet frozenPatterns(std::move(patterns));
     while (succeeded(fuseLinalgOpsGreedily(getFunction()))) {
       (void)applyPatternsAndFoldGreedily(getFunction(), frozenPatterns);
       PassManager pm(context);
       pm.addPass(createLoopInvariantCodeMotionPass());
       pm.addPass(createCanonicalizerPass());
       pm.addPass(createCSEPass());
       LogicalResult res = pm.run(getFunction()->getParentOfType<ModuleOp>());
       if (failed(res))
         this->signalPassFailure();
     }
   }
 };
 
 /// Pass to test tile and fuse of sequence of operations. Intended only for
 /// testing.
 struct TestLinalgTileAndFuseSequencePass
     : public PassWrapper<TestLinalgTileAndFuseSequencePass, FunctionPass> {
   TestLinalgTileAndFuseSequencePass() = default;
   TestLinalgTileAndFuseSequencePass(
       const TestLinalgTileAndFuseSequencePass &pass){};
 
   ListOption<int64_t> tileSizes{
       *this, "tile-sizes", llvm::cl::desc("Tile sizes to use for ops"),
       llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated};
 
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<AffineDialect, linalg::LinalgDialect, memref::MemRefDialect,
                     scf::SCFDialect>();
   }
 
   void runOnFunction() override {
     FuncOp funcOp = getOperation();
     auto &blocks = funcOp.getBody().getBlocks();
     if (!llvm::hasSingleElement(blocks)) {
       return;
     }
     SmallVector<LinalgOp, 2> linalgOps =
         llvm::to_vector<2>(blocks.front().getOps<LinalgOp>());
     Aliases aliases;
     LinalgDependenceGraph dependenceGraph(aliases, linalgOps);
     OpBuilder builder(funcOp.getContext());
     linalg::LinalgTilingLoopType loopType = LinalgTilingLoopType::ParallelLoops;
     if (llvm::any_of(linalgOps, [](LinalgOp linalgOp) {
           return linalgOp.hasTensorSemantics();
         }))
       loopType = LinalgTilingLoopType::Loops;
     Optional<TiledAndFusedLinalgOps> tileAndFuseOps = tileAndFuseLinalgOps(
         builder, linalgOps, dependenceGraph,
         LinalgTilingOptions().setTileSizes(tileSizes).setLoopType(loopType));
     if (!tileAndFuseOps)
       return signalPassFailure();
     if (linalgOps.back().hasTensorSemantics()) {
       linalgOps.back().getOperation()->replaceAllUsesWith(
           tileAndFuseOps->fusedLoops.front());
     }
     for (auto op : linalgOps)
       if (op.hasBufferSemantics())
         op.erase();
   }
 };
 } // namespace
 
 namespace mlir {
 namespace test {
 void registerTestLinalgFusionTransforms() {
   PassRegistration<TestLinalgFusionTransforms<>> testFusionTransformsPass(
       "test-linalg-fusion-transform-patterns",
       "Test Linalg fusion transformation patterns by applying them greedily.");
 }
 void registerTestLinalgTensorFusionTransforms() {
   PassRegistration<TestLinalgFusionTransforms<LinalgTilingLoopType::Loops>>
       testTensorFusionTransformsPass(
           "test-linalg-tensor-fusion-transform-patterns",
           "Test Linalg on tensor fusion transformation "
           "patterns by applying them greedily.");
 }
 void registerTestLinalgGreedyFusion() {
   PassRegistration<TestLinalgGreedyFusion> testFusionTransformsPass(
       "test-linalg-greedy-fusion",
       "Test Linalg fusion by applying a greedy test transformation.");
 }
 void registerTestLinalgTileAndFuseSequencePass() {
   PassRegistration<TestLinalgTileAndFuseSequencePass>
       testTileAndFuseSequencePass(
           "test-linalg-tile-and-fuse",
           "Test Linalg tiling and fusion of a sequence of Linalg operations.");
 }
 
 } // namespace test
 } // namespace mlir

mlir/test/lib/Dialect/Linalg/TestLinalgHoisting.cpp

 //===- TestLinalgHoisting.cpp - Test Linalg hoisting functions ------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements logic for testing Linalg hoisting functions.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Linalg/IR/LinalgOps.h"
 #include "mlir/Dialect/Linalg/Transforms/Hoisting.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 using namespace mlir::linalg;
 
 namespace {
 struct TestLinalgHoisting
     : public PassWrapper<TestLinalgHoisting, FunctionPass> {
   TestLinalgHoisting() = default;
   TestLinalgHoisting(const TestLinalgHoisting &pass) {}
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<AffineDialect>();
   }
 
   void runOnFunction() override;
 
   Option<bool> testHoistRedundantTransfers{
       *this, "test-hoist-redundant-transfers",
       llvm::cl::desc("Test hoisting transfer_read/transfer_write pairs"),
       llvm::cl::init(false)};
 };
 } // end anonymous namespace
 
 void TestLinalgHoisting::runOnFunction() {
   if (testHoistRedundantTransfers) {
     hoistRedundantVectorTransfers(getFunction());
     hoistRedundantVectorTransfersOnTensor(getFunction());
     return;
   }
 }
 
 namespace mlir {
 namespace test {
 void registerTestLinalgHoisting() {
   PassRegistration<TestLinalgHoisting> testTestLinalgHoistingPass(
       "test-linalg-hoisting", "Test Linalg hoisting functions.");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Dialect/Linalg/TestLinalgTransforms.cpp

 //===- TestLinalgTransforms.cpp - Test Linalg transformation patterns -----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements logic for testing Linalg transformations.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/GPU/GPUDialect.h"
 #include "mlir/Dialect/Linalg/IR/LinalgOps.h"
 #include "mlir/Dialect/Linalg/Transforms/Hoisting.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 #include "llvm/ADT/SetVector.h"
 
 using namespace mlir;
 using namespace mlir::linalg;
 
 namespace {
 struct TestLinalgTransforms
     : public PassWrapper<TestLinalgTransforms, FunctionPass> {
   TestLinalgTransforms() = default;
   TestLinalgTransforms(const TestLinalgTransforms &pass) {}
 
   void getDependentDialects(DialectRegistry &registry) const override {
     // clang-format off
     registry.insert<AffineDialect,
                     memref::MemRefDialect,
                     scf::SCFDialect,
                     StandardOpsDialect,
                     vector::VectorDialect,
                     gpu::GPUDialect>();
     // clang-format on
   }
 
   void runOnFunction() override;
 
   Option<bool> testPatterns{*this, "test-patterns",
                             llvm::cl::desc("Test a mixed set of patterns"),
                             llvm::cl::init(false)};
   Option<bool> testMatmulToVectorPatterns1dTiling{
       *this, "test-matmul-to-vector-patterns-tile-1d",
       llvm::cl::desc(
           "Test a fused pass that applies patterns from matmul to vectors via "
           "1-d tiling"),
       llvm::cl::init(false)};
   Option<bool> testMatmulToVectorPatterns2dTiling{
       *this, "test-matmul-to-vector-patterns-tile-2d",
       llvm::cl::desc(
           "Test a fused pass that applies patterns from matmul to vectors via "
           "2-d tiling"),
       llvm::cl::init(false)};
   Option<bool> testPromotionOptions{*this, "test-linalg-promotion-options",
                                     llvm::cl::desc("Test promotion options"),
                                     llvm::cl::init(false)};
   Option<bool> testTileAndDistributionOptions{
       *this, "test-tile-and-distribute-options",
       llvm::cl::desc("Test tile and distribute options"),
       llvm::cl::init(false)};
   Option<bool> testVectorTransferForwardingPatterns{
       *this, "test-vector-transfer-forwarding-patterns",
       llvm::cl::desc(
           "Test a fused pass that forwards linalg.copy to vector.transfer"),
       llvm::cl::init(false)};
   Option<bool> testGenericToVectorPattern{
       *this, "test-linalg-to-vector-patterns",
       llvm::cl::desc("Test a set of patterns that rewrite a linalg contraction "
                      "in vector.contract form"),
       llvm::cl::init(false)};
   Option<bool> testAffineMinSCFCanonicalizationPatterns{
       *this, "test-affine-min-scf-canonicalization-patterns",
       llvm::cl::desc("Test affine-min + scf canonicalization patterns."),
       llvm::cl::init(false)};
   Option<bool> testTileAndPadPattern{
       *this, "test-tile-and-pad-pattern",
       llvm::cl::desc("Test tile and pad pattern"), llvm::cl::init(false)};
   Option<int> testHoistPadding{*this, "test-hoist-padding",
                                llvm::cl::desc("Test hoist padding"),
                                llvm::cl::init(0)};
   ListOption<int64_t> tileSizesForPadding{
       *this, "tile-sizes-for-padding",
       llvm::cl::desc("Linalg tile sizes when tile+pad"), llvm::cl::ZeroOrMore,
       llvm::cl::MiscFlags::CommaSeparated};
   ListOption<unsigned> testInterchangePattern{
       *this, "test-interchange-pattern", llvm::cl::MiscFlags::CommaSeparated,
       llvm::cl::desc("Test the interchange pattern.")};
 };
 } // end anonymous namespace
 
 static void applyPatterns(FuncOp funcOp) {
   MLIRContext *ctx = funcOp.getContext();
   RewritePatternSet patterns(ctx);
 
   //===--------------------------------------------------------------------===//
   // Linalg tiling patterns.
   //===--------------------------------------------------------------------===//
   patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx, LinalgTilingOptions().setTileSizes({2000, 3000, 4000}),
       LinalgTransformationFilter(Identifier::get("MEM", ctx),
                                  Identifier::get("L3", ctx)));
   patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx, LinalgTilingOptions().setTileSizes({200, 300, 400}),
       LinalgTransformationFilter(Identifier::get("L3", ctx),
                                  Identifier::get("L2", ctx)));
   patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx, LinalgTilingOptions().setTileSizes({20, 30, 40}),
       LinalgTransformationFilter(Identifier::get("L2", ctx),
                                  Identifier::get("L1", ctx)));
   patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx, LinalgTilingOptions().setTileSizes({2, 3, 4}),
       LinalgTransformationFilter(Identifier::get("L1", ctx),
                                  Identifier::get("REG", ctx)));
 
   patterns.add<LinalgTilingPattern<MatvecOp>>(
       ctx,
       LinalgTilingOptions().setTileSizes({5, 6}).setLoopType(
           LinalgTilingLoopType::ParallelLoops),
       LinalgTransformationFilter(ArrayRef<Identifier>{},
                                  Identifier::get("L1", ctx)));
 
   patterns.add<LinalgTilingPattern<DotOp>>(
       ctx, LinalgTilingOptions().setTileSizes(8000),
       LinalgTransformationFilter(
           ArrayRef<Identifier>{Identifier::get("MEM", ctx),
                                Identifier::get("L3", ctx),
                                Identifier::get("L2", ctx)},
           Identifier::get("REG", ctx)));
 
   //===--------------------------------------------------------------------===//
   // Linalg tiling and permutation patterns.
   //===--------------------------------------------------------------------===//
   patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx,
       LinalgTilingOptions()
           .setTileSizes({2000, 3000, 4000})
           .setInterchange({1, 2, 0}),
       LinalgTransformationFilter(Identifier::get("__with_perm__", ctx),
                                  Identifier::get("L2__with_perm__", ctx)));
   patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx,
       LinalgTilingOptions()
           .setTileSizes({200, 300, 400})
           .setInterchange({1, 0, 2}),
       LinalgTransformationFilter(Identifier::get("L2__with_perm__", ctx),
                                  Identifier::get("L1__with_perm__", ctx)));
   patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx, LinalgTilingOptions().setTileSizes({20, 30, 40}),
       LinalgTransformationFilter(Identifier::get("L1__with_perm__", ctx),
                                  Identifier::get("REG__with_perm__", ctx)));
 
   patterns.add<LinalgTilingPattern<MatvecOp>>(
       ctx, LinalgTilingOptions().setTileSizes({5, 6}).setInterchange({1, 0}),
       LinalgTransformationFilter(Identifier::get("__with_perm__", ctx),
                                  Identifier::get("L1__with_perm__", ctx)));
 
   patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx,
       LinalgTilingOptions()
           .setTileSizes({16, 8, 4})
           .setInterchange({1, 2, 0})
           .setLoopType(LinalgTilingLoopType::ParallelLoops),
       LinalgTransformationFilter(
           Identifier::get("par__with_perm__", ctx),
           Identifier::get("after_par__with_perm__", ctx)));
 
   //===--------------------------------------------------------------------===//
   // Linalg to loops patterns.
   //===--------------------------------------------------------------------===//
   patterns.add<LinalgLoweringPattern<DotOp>>(
       ctx,
       /*loweringType=*/LinalgLoweringType::Loops,
       LinalgTransformationFilter(Identifier::get("REG", ctx)));
 
   //===--------------------------------------------------------------------===//
   // Linalg distribution patterns.
   //===--------------------------------------------------------------------===//
   LinalgLoopDistributionOptions distributionOptions;
 
   //===--------------------------------------------------------------------===//
   // Linalg to vector contraction patterns.
   //===--------------------------------------------------------------------===//
   patterns.add<LinalgVectorizationPattern>(
       ctx, LinalgTransformationFilter(Identifier::get("VECTORIZE", ctx))
                .addOpFilter<MatmulOp, FillOp, CopyOp, GenericOp>());
 
   //===--------------------------------------------------------------------===//
   // Linalg generic permutation patterns.
   //===--------------------------------------------------------------------===//
   patterns.add<LinalgInterchangePattern<GenericOp>>(
       ctx,
       /*interchangeVector=*/ArrayRef<unsigned>{1, 2, 0},
       LinalgTransformationFilter(ArrayRef<Identifier>{},
                                  Identifier::get("PERMUTED", ctx)));
   patterns.add<LinalgInterchangePattern<IndexedGenericOp>>(
       ctx,
       /*interchangeVector=*/ArrayRef<unsigned>{1, 2, 0},
       LinalgTransformationFilter(ArrayRef<Identifier>{},
                                  Identifier::get("PERMUTED", ctx)));
 
   //===--------------------------------------------------------------------===//
   // Linalg subview operands promotion.
   //===--------------------------------------------------------------------===//
   patterns.add<LinalgPromotionPattern<MatmulOp>>(
       ctx, LinalgPromotionOptions().setUseFullTileBuffersByDefault(true),
       LinalgTransformationFilter(Identifier::get("_promote_views_", ctx),
                                  Identifier::get("_views_promoted_", ctx)));
   patterns.add<LinalgPromotionPattern<MatmulOp>>(
       ctx,
       LinalgPromotionOptions()
           .setOperandsToPromote({0})
           .setUseFullTileBuffersByDefault(true),
       LinalgTransformationFilter(
           Identifier::get("_promote_first_view_", ctx),
           Identifier::get("_first_view_promoted_", ctx)));
   patterns.add<LinalgPromotionPattern<FillOp>>(
       ctx,
       LinalgPromotionOptions()
           .setOperandsToPromote({0})
           .setUseFullTileBuffers({true})
           .setAlignment(32),
       LinalgTransformationFilter(
           Identifier::get("_promote_views_aligned_", ctx),
           Identifier::get("_views_aligned_promoted_", ctx)));
 
   (void)applyPatternsAndFoldGreedily(funcOp, std::move(patterns));
 
   // Drop the marker.
   funcOp.walk([](LinalgOp op) {
     op->removeAttr(LinalgTransforms::kLinalgTransformMarker);
   });
 }
 
 static void fillL1TilingAndMatmulToVectorPatterns(
     FuncOp funcOp, StringRef startMarker,
     SmallVectorImpl<RewritePatternSet> &patternsVector) {
   MLIRContext *ctx = funcOp.getContext();
   patternsVector.emplace_back(
       ctx, std::make_unique<LinalgTilingPattern<MatmulOp>>(
                ctx,
                LinalgTilingOptions()
                    .setTileSizes({8, 12, 16})
                    .setInterchange({1, 0, 2}),
                LinalgTransformationFilter(Identifier::get(startMarker, ctx),
                                           Identifier::get("L1", ctx))));
 
   patternsVector.emplace_back(
       ctx,
       std::make_unique<LinalgPromotionPattern<MatmulOp>>(
           ctx, LinalgPromotionOptions().setUseFullTileBuffersByDefault(true),
           LinalgTransformationFilter(Identifier::get("L1", ctx),
                                      Identifier::get("VEC", ctx))));
 
   patternsVector.emplace_back(
       ctx, std::make_unique<LinalgVectorizationPattern>(
                MatmulOp::getOperationName(), ctx, LinalgVectorizationOptions(),
                LinalgTransformationFilter(Identifier::get("VEC", ctx))));
   patternsVector.back().add<LinalgVectorizationPattern>(
       ctx, LinalgTransformationFilter().addFilter(
                [](Operation *op) { return success(isa<FillOp, CopyOp>(op)); }));
 }
 
 //===----------------------------------------------------------------------===//
 // Test promotion callbacks
 //===----------------------------------------------------------------------===//
 
 // Allocation call back
 static Optional<Value> allocCallBackFn(OpBuilder &b, memref::SubViewOp subView,
                                        ArrayRef<Value> boundingSubViewSize,
                                        DataLayout &layout,
                                        OperationFolder *folder) {
   SmallVector<int64_t, 4> shape(boundingSubViewSize.size(), -1);
   return b
       .create<memref::AllocOp>(
           subView.getLoc(),
           MemRefType::get(shape, subView.getType().getElementType(),
                           /*affineMapComposition =*/{}, 3),
           boundingSubViewSize)
       .getResult();
 }
 
 // Deallocation callback
 static LogicalResult deallocCallBackFn(OpBuilder &b, Value buffer) {
   b.create<memref::DeallocOp>(buffer.getLoc(), buffer);
   return success();
 }
 
 // Copy in call back
 static LogicalResult copyCallBackFn(OpBuilder &b, Value src, Value dst,
                                     bool isOutput) {
   auto floatType = src.getType().cast<MemRefType>().getElementType();
   if (!floatType.isa<FloatType>())
     return failure();
   if (!isOutput)
     b.create<FillOp>(
         src.getLoc(), dst,
         b.create<ConstantOp>(src.getLoc(), FloatAttr::get(floatType, 42.0)));
   b.create<CopyOp>(src.getLoc(), src, dst);
   return success();
 }
 
 static void fillPromotionCallBackPatterns(MLIRContext *ctx,
                                           RewritePatternSet &patterns) {
   patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx, LinalgTilingOptions().setTileSizes({16, 16, 16}),
       LinalgTransformationFilter(Identifier::get("START", ctx),
                                  Identifier::get("PROMOTE", ctx)));
   patterns.add<LinalgPromotionPattern<MatmulOp>>(
       ctx,
       LinalgPromotionOptions()
           .setOperandsToPromote({0, 2})
           .setUseFullTileBuffers({false, false})
           .setAllocationDeallocationFns(allocCallBackFn, deallocCallBackFn)
           .setCopyInOutFns(
               [](OpBuilder &b, Value src, Value dst) -> LogicalResult {
                 return copyCallBackFn(b, src, dst, false);
               },
               [](OpBuilder &b, Value src, Value dst) -> LogicalResult {
                 return copyCallBackFn(b, src, dst, true);
               }),
       LinalgTransformationFilter(Identifier::get("PROMOTE", ctx)));
 }
 
 template <typename IdOp, typename NProcsOp>
 static SmallVector<ProcInfo, 2>
 getGpuProcIds(OpBuilder &b, Location loc, ArrayRef<Range> parallelLoopRanges) {
   size_t count = std::min<size_t>(3, parallelLoopRanges.size());
   SmallVector<ProcInfo, 2> procInfo(count);
   const char *xyz[] = {"x", "y", "z"};
   Type indexType = b.getIndexType();
   for (unsigned i = 0; i < count; ++i) {
     procInfo[count - 1 - i] = {
         b.create<IdOp>(loc, indexType, b.getStringAttr(xyz[i])),
         b.create<NProcsOp>(loc, indexType, b.getStringAttr(xyz[i]))};
   }
   return procInfo;
 }
 
 static void fillTileAndDistributePatterns(MLIRContext *context,
                                           RewritePatternSet &patterns) {
   {
     LinalgLoopDistributionOptions cyclicNprocsEqNiters;
     cyclicNprocsEqNiters.distributionMethod.resize(
         2, DistributionMethod::CyclicNumProcsEqNumIters);
     cyclicNprocsEqNiters.procInfo =
         getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
     patterns.add<LinalgTilingPattern<MatmulOp>>(
         context,
         LinalgTilingOptions()
             .setTileSizes({8, 8, 4})
             .setLoopType(LinalgTilingLoopType::ParallelLoops)
             .setDistributionOptions(cyclicNprocsEqNiters),
         LinalgTransformationFilter(
             Identifier::get("distribute1", context),
             Identifier::get("after_distribute1", context)));
   }
 
   {
     LinalgLoopDistributionOptions cyclicNprocsGeNiters;
     cyclicNprocsGeNiters.distributionMethod.resize(
         2, DistributionMethod::CyclicNumProcsGeNumIters);
     cyclicNprocsGeNiters.procInfo =
         getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
     patterns.add<LinalgTilingPattern<MatmulOp>>(
         context,
         LinalgTilingOptions()
             .setTileSizes({8, 8, 4})
             .setLoopType(LinalgTilingLoopType::ParallelLoops)
             .setDistributionOptions(cyclicNprocsGeNiters),
         LinalgTransformationFilter(
             Identifier::get("distribute2", context),
             Identifier::get("after_distribute2", context)));
   }
 
   {
     LinalgLoopDistributionOptions cyclicNprocsDefault;
     cyclicNprocsDefault.distributionMethod.resize(2,
                                                   DistributionMethod::Cyclic);
     cyclicNprocsDefault.procInfo =
         getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
     patterns.add<LinalgTilingPattern<MatmulOp>>(
         context,
         LinalgTilingOptions()
             .setTileSizes({8, 8, 4})
             .setLoopType(LinalgTilingLoopType::ParallelLoops)
             .setDistributionOptions(cyclicNprocsDefault),
         LinalgTransformationFilter(
             Identifier::get("distribute3", context),
             Identifier::get("after_distribute3", context)));
   }
 
   {
     LinalgLoopDistributionOptions cyclicNprocsMixed1;
     cyclicNprocsMixed1.distributionMethod = {
         DistributionMethod::CyclicNumProcsEqNumIters,
         DistributionMethod::CyclicNumProcsGeNumIters};
     cyclicNprocsMixed1.procInfo = getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
     patterns.add<LinalgTilingPattern<MatmulOp>>(
         context,
         LinalgTilingOptions()
             .setTileSizes({8, 8, 4})
             .setLoopType(LinalgTilingLoopType::ParallelLoops)
             .setDistributionOptions(cyclicNprocsMixed1),
         LinalgTransformationFilter(
             Identifier::get("distribute4", context),
             Identifier::get("after_distribute4", context)));
   }
 
   {
     LinalgLoopDistributionOptions cyclicNprocsMixed2;
     cyclicNprocsMixed2.distributionMethod = {
         DistributionMethod::CyclicNumProcsGeNumIters,
         DistributionMethod::Cyclic};
     cyclicNprocsMixed2.procInfo = getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
     patterns.add<LinalgTilingPattern<MatmulOp>>(
         context,
         LinalgTilingOptions()
             .setTileSizes({8, 8, 4})
             .setLoopType(LinalgTilingLoopType::ParallelLoops)
             .setDistributionOptions(cyclicNprocsMixed2),
         LinalgTransformationFilter(
             Identifier::get("distribute5", context),
             Identifier::get("after_distribute5", context)));
   }
 
   {
     LinalgLoopDistributionOptions cyclicNprocsMixed3;
     cyclicNprocsMixed3.distributionMethod = {
         DistributionMethod::Cyclic,
         DistributionMethod::CyclicNumProcsEqNumIters};
     cyclicNprocsMixed3.procInfo = getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
 
     patterns.add<LinalgTilingPattern<MatmulOp>>(
         context,
         LinalgTilingOptions()
             .setTileSizes({8, 8, 4})
             .setLoopType(LinalgTilingLoopType::ParallelLoops)
             .setDistributionOptions(cyclicNprocsMixed3),
         LinalgTransformationFilter(
             Identifier::get("distribute6", context),
             Identifier::get("after_distribute6", context)));
   }
 
   {
     LinalgLoopDistributionOptions cyclicNprocsEqNiters;
     cyclicNprocsEqNiters.distributionMethod.resize(2,
                                                    DistributionMethod::Cyclic);
     cyclicNprocsEqNiters.procInfo =
         getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
     patterns.add<LinalgTilingPattern<MatmulOp>>(
         context,
         LinalgTilingOptions()
             .setTileSizes({8, 8, 4})
             .setLoopType(LinalgTilingLoopType::Loops)
             .setDistributionOptions(cyclicNprocsEqNiters),
         LinalgTransformationFilter(
             Identifier::get("tensors_distribute1", context),
             Identifier::get("tensors_after_distribute1", context)));
   }
 }
 
 static void
 applyMatmulToVectorPatterns(FuncOp funcOp,
                             bool testMatmulToVectorPatterns1dTiling,
                             bool testMatmulToVectorPatterns2dTiling) {
   MLIRContext *ctx = funcOp.getContext();
   SmallVector<RewritePatternSet, 4> stage1Patterns;
   if (testMatmulToVectorPatterns1dTiling) {
     fillL1TilingAndMatmulToVectorPatterns(funcOp, Identifier::get("START", ctx),
                                           stage1Patterns);
   } else if (testMatmulToVectorPatterns2dTiling) {
     stage1Patterns.emplace_back(
         ctx, std::make_unique<LinalgTilingPattern<MatmulOp>>(
                  ctx,
                  LinalgTilingOptions()
                      .setTileSizes({768, 264, 768})
                      .setInterchange({1, 2, 0}),
                  LinalgTransformationFilter(Identifier::get("START", ctx),
                                             Identifier::get("L2", ctx))));
     fillL1TilingAndMatmulToVectorPatterns(funcOp, Identifier::get("L2", ctx),
                                           stage1Patterns);
   }
   SmallVector<FrozenRewritePatternSet, 4> frozenStage1Patterns;
   llvm::move(stage1Patterns, std::back_inserter(frozenStage1Patterns));
   FrozenRewritePatternSet stage2Patterns =
       getLinalgTilingCanonicalizationPatterns(ctx);
   (void)applyStagedPatterns(funcOp, frozenStage1Patterns,
                             std::move(stage2Patterns));
 }
 
 static void applyVectorTransferForwardingPatterns(FuncOp funcOp) {
   RewritePatternSet forwardPattern(funcOp.getContext());
   forwardPattern.add<LinalgCopyVTRForwardingPattern>(funcOp.getContext());
   forwardPattern.add<LinalgCopyVTWForwardingPattern>(funcOp.getContext());
   (void)applyPatternsAndFoldGreedily(funcOp, std::move(forwardPattern));
 }
 
 static void applyLinalgToVectorPatterns(FuncOp funcOp) {
   RewritePatternSet patterns(funcOp.getContext());
   patterns.add<LinalgVectorizationPattern>(
       funcOp.getContext(),
       LinalgTransformationFilter()
           .addOpFilter<ContractionOpInterface, FillOp, CopyOp, GenericOp>());
   patterns.add<PadTensorOpVectorizationPattern>(funcOp.getContext());
   (void)applyPatternsAndFoldGreedily(funcOp, std::move(patterns));
 }
 
 static void applyAffineMinSCFCanonicalizationPatterns(FuncOp funcOp) {
   RewritePatternSet foldPattern(funcOp.getContext());
   foldPattern.add<AffineMinSCFCanonicalizationPattern>(funcOp.getContext());
   FrozenRewritePatternSet frozenPatterns(std::move(foldPattern));
 
   // Explicitly walk and apply the pattern locally to avoid more general folding
   // on the rest of the IR.
   funcOp.walk([&frozenPatterns](AffineMinOp minOp) {
     (void)applyOpPatternsAndFold(minOp, frozenPatterns);
   });
 }
 
 // For now, just assume it is the zero of type.
 // In the future, it should be the zero of type + op.
 static Value getNeutralOfLinalgOp(OpBuilder &b, OpOperand &op) {
   auto t = getElementTypeOrSelf(op.get().getType());
   return b.create<ConstantOp>(op.getOwner()->getLoc(), t, b.getZeroAttr(t));
 }
 
 static void applyTileAndPadPattern(FuncOp funcOp, ArrayRef<int64_t> tileSizes) {
   MLIRContext *context = funcOp.getContext();
   RewritePatternSet tilingPattern(context);
   auto linalgTilingOptions =
       linalg::LinalgTilingOptions()
           .setTileSizes(tileSizes)
           .setPaddingValueComputationFunction(getNeutralOfLinalgOp);
   tilingPattern.add<linalg::LinalgTilingPattern<linalg::MatmulI8I8I32Op>>(
       context, linalgTilingOptions,
       linalg::LinalgTransformationFilter(
           Identifier::get("tile-and-pad", context)));
   (void)applyPatternsAndFoldGreedily(funcOp, std::move(tilingPattern));
 }
 
 static void applyInterchangePattern(FuncOp funcOp,
                                     ArrayRef<unsigned> interchangeVector) {
   MLIRContext *context = funcOp.getContext();
   RewritePatternSet interchangePattern(context);
   interchangePattern.add<LinalgInterchangePattern<GenericOp>>(
       context, interchangeVector,
       LinalgTransformationFilter(ArrayRef<Identifier>{},
                                  Identifier::get("interchange", context)));
   (void)applyPatternsAndFoldGreedily(funcOp, std::move(interchangePattern));
 }
 
 /// Apply transformations specified as patterns.
 void TestLinalgTransforms::runOnFunction() {
   auto lambda = [&](void *) {
     getFunction().walk([](LinalgOp op) {
       op->removeAttr(LinalgTransforms::kLinalgTransformMarker);
     });
   };
   std::unique_ptr<void, decltype(lambda)> cleanupGuard{(void *)1, lambda};
 
   if (testPromotionOptions) {
     RewritePatternSet patterns(&getContext());
     fillPromotionCallBackPatterns(&getContext(), patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
     return;
   }
   if (testTileAndDistributionOptions) {
     RewritePatternSet patterns(&getContext());
     fillTileAndDistributePatterns(&getContext(), patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
     return;
   }
   if (testPatterns)
     return applyPatterns(getFunction());
   if (testMatmulToVectorPatterns1dTiling || testMatmulToVectorPatterns2dTiling)
     return applyMatmulToVectorPatterns(getFunction(),
                                        testMatmulToVectorPatterns1dTiling,
                                        testMatmulToVectorPatterns2dTiling);
   if (testVectorTransferForwardingPatterns)
     return applyVectorTransferForwardingPatterns(getFunction());
   if (testGenericToVectorPattern)
     return applyLinalgToVectorPatterns(getFunction());
   if (testAffineMinSCFCanonicalizationPatterns)
     return applyAffineMinSCFCanonicalizationPatterns(getFunction());
   if (testTileAndPadPattern)
     return applyTileAndPadPattern(getFunction(), tileSizesForPadding);
   if (testHoistPadding) {
     getFunction().walk([&](linalg::PadTensorOp padTensorOp) {
       (void)linalg::hoistPaddingOnTensors(padTensorOp, testHoistPadding);
     });
   }
   if (testInterchangePattern.hasValue())
     return applyInterchangePattern(getFunction(), testInterchangePattern);
 }
 
 namespace mlir {
 namespace test {
 void registerTestLinalgTransforms() {
   PassRegistration<TestLinalgTransforms> testTransformPatternsPass(
       "test-linalg-transform-patterns",
       "Test Linalg transformation patterns by applying them greedily.");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Dialect/Math/CMakeLists.txt

+# Exclude tests from libMLIR.so
+add_mlir_library(MLIRMathTestPasses
+  TestExpandTanh.cpp
+  TestPolynomialApproximation.cpp
+
+  EXCLUDE_FROM_LIBMLIR
+
+  LINK_LIBS PUBLIC
+  MLIRMathTransforms
+  MLIRPass
+  MLIRTransformUtils
+  MLIRVector
+  )

mlir/test/lib/Dialect/Math/TestExpandTanh.cpp

 //===- TestExpandTanh.cpp - Test expand tanh op into exp form -------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains test passes for expanding tanh.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Math/Transforms/Passes.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
 
 namespace {
 struct TestExpandTanhPass
     : public PassWrapper<TestExpandTanhPass, FunctionPass> {
   void runOnFunction() override;
 };
 } // end anonymous namespace
 
 void TestExpandTanhPass::runOnFunction() {
   RewritePatternSet patterns(&getContext());
   populateExpandTanhPattern(patterns);
   (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
 }
 
 namespace mlir {
 namespace test {
 void registerTestExpandTanhPass() {
   PassRegistration<TestExpandTanhPass> pass("test-expand-tanh",
                                             "Test expanding tanh");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Dialect/Math/TestPolynomialApproximation.cpp

 //===- TestPolynomialApproximation.cpp - Test math ops approximations -----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains test passes for expanding math operations into
 // polynomial approximations.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/Dialect/Math/Transforms/Passes.h"
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
 
 namespace {
 struct TestMathPolynomialApproximationPass
     : public PassWrapper<TestMathPolynomialApproximationPass, FunctionPass> {
   void runOnFunction() override;
   void getDependentDialects(DialectRegistry &registry) const override {
     registry
         .insert<vector::VectorDialect, math::MathDialect, LLVM::LLVMDialect>();
   }
 };
 } // end anonymous namespace
 
 void TestMathPolynomialApproximationPass::runOnFunction() {
   RewritePatternSet patterns(&getContext());
   populateMathPolynomialApproximationPatterns(patterns);
   (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
 }
 
 namespace mlir {
 namespace test {
 void registerTestMathPolynomialApproximationPass() {
   PassRegistration<TestMathPolynomialApproximationPass> pass(
       "test-math-polynomial-approximation",
       "Test math polynomial approximations");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Dialect/SCF/CMakeLists.txt

+# Exclude tests from libMLIR.so
+add_mlir_library(MLIRSCFTestPasses
+  TestSCFUtils.cpp
+
+  EXCLUDE_FROM_LIBMLIR
+
+  LINK_LIBS PUBLIC
+  MLIRPass
+  MLIRSCF
+  MLIRSCFTransforms
+  MLIRTransformUtils
+  )

mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp

 //===- TestSCFUtils.cpp --- Pass to test independent SCF dialect utils ----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements a pass to test SCF dialect utils.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/SCF/SCF.h"
 #include "mlir/Dialect/SCF/Utils.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/Passes.h"
 
 #include "llvm/ADT/SetVector.h"
 
 using namespace mlir;
 
 namespace {
 class TestSCFForUtilsPass
     : public PassWrapper<TestSCFForUtilsPass, FunctionPass> {
 public:
   explicit TestSCFForUtilsPass() {}
 
   void runOnFunction() override {
     FuncOp func = getFunction();
     SmallVector<scf::ForOp, 4> toErase;
 
     func.walk([&](Operation *fakeRead) {
       if (fakeRead->getName().getStringRef() != "fake_read")
         return;
       auto *fakeCompute = fakeRead->getResult(0).use_begin()->getOwner();
       auto *fakeWrite = fakeCompute->getResult(0).use_begin()->getOwner();
       auto loop = fakeRead->getParentOfType<scf::ForOp>();
 
       OpBuilder b(loop);
       (void)loop.moveOutOfLoop({fakeRead});
       fakeWrite->moveAfter(loop);
       auto newLoop = cloneWithNewYields(b, loop, fakeRead->getResult(0),
                                         fakeCompute->getResult(0));
       fakeCompute->getResult(0).replaceAllUsesWith(
           newLoop.getResults().take_back()[0]);
       toErase.push_back(loop);
     });
     for (auto loop : llvm::reverse(toErase))
       loop.erase();
   }
 };
 
 class TestSCFIfUtilsPass
     : public PassWrapper<TestSCFIfUtilsPass, FunctionPass> {
 public:
   explicit TestSCFIfUtilsPass() {}
 
   void runOnFunction() override {
     int count = 0;
     FuncOp func = getFunction();
     func.walk([&](scf::IfOp ifOp) {
       auto strCount = std::to_string(count++);
       FuncOp thenFn, elseFn;
       OpBuilder b(ifOp);
       outlineIfOp(b, ifOp, &thenFn, std::string("outlined_then") + strCount,
                   &elseFn, std::string("outlined_else") + strCount);
     });
   }
 };
 } // namespace
 
 namespace mlir {
 namespace test {
 void registerTestSCFUtilsPass() {
   PassRegistration<TestSCFForUtilsPass>("test-scf-for-utils",
                                         "test scf.for utils");
   PassRegistration<TestSCFIfUtilsPass>("test-scf-if-utils",
                                        "test scf.if utils");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Dialect/StandardOps/CMakeLists.txt

+# Exclude tests from libMLIR.so
+add_mlir_library(MLIRStandardOpsTestPasses
+  TestDecomposeCallGraphTypes.cpp
+  TestComposeSubView.cpp
+
+  EXCLUDE_FROM_LIBMLIR
+
+  LINK_LIBS PUBLIC
+  MLIRAffine
+  MLIRPass
+  MLIRStandardOpsTransforms
+  MLIRTestDialect
+  MLIRTransformUtils
+  )
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../Test)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/../Test)

mlir/test/lib/Dialect/StandardOps/TestComposeSubView.cpp

 //===- TestComposeSubView.cpp - Test composed subviews --------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements a pass to test the composed subview patterns.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/StandardOps/Transforms/ComposeSubView.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
 
 namespace {
 struct TestComposeSubViewPass
     : public PassWrapper<TestComposeSubViewPass, FunctionPass> {
   void runOnFunction() override;
   void getDependentDialects(DialectRegistry &registry) const override;
 };
 
 void TestComposeSubViewPass::getDependentDialects(
     DialectRegistry &registry) const {
   registry.insert<AffineDialect>();
 }
 
 void TestComposeSubViewPass::runOnFunction() {
   OwningRewritePatternList patterns(&getContext());
   populateComposeSubViewPatterns(patterns, &getContext());
   (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
 }
 } // namespace
 
 namespace mlir {
 namespace test {
 void registerTestComposeSubView() {
   PassRegistration<TestComposeSubViewPass> pass(
       "test-compose-subview", "Test combining composed subviews");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Dialect/StandardOps/TestDecomposeCallGraphTypes.cpp

 //===- TestDecomposeCallGraphTypes.cpp - Test CG type decomposition -------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "TestDialect.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/StandardOps/Transforms/DecomposeCallGraphTypes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 using namespace mlir;
 
 namespace {
 /// A pass for testing call graph type decomposition.
 ///
 /// This instantiates the patterns with a TypeConverter and ValueDecomposer
 /// that splits tuple types into their respective element types.
 /// For example, `tuple<T1, T2, T3> --> T1, T2, T3`.
 struct TestDecomposeCallGraphTypes
     : public PassWrapper<TestDecomposeCallGraphTypes, OperationPass<ModuleOp>> {
 
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<test::TestDialect>();
   }
   void runOnOperation() override {
     ModuleOp module = getOperation();
     auto *context = &getContext();
     TypeConverter typeConverter;
     ConversionTarget target(*context);
     ValueDecomposer decomposer;
     RewritePatternSet patterns(context);
 
     target.addLegalDialect<test::TestDialect>();
 
     target.addDynamicallyLegalOp<ReturnOp>([&](ReturnOp op) {
       return typeConverter.isLegal(op.getOperandTypes());
     });
     target.addDynamicallyLegalOp<CallOp>(
         [&](CallOp op) { return typeConverter.isLegal(op); });
     target.addDynamicallyLegalOp<FuncOp>([&](FuncOp op) {
       return typeConverter.isSignatureLegal(op.getType());
     });
 
     typeConverter.addConversion([](Type type) { return type; });
     typeConverter.addConversion(
         [](TupleType tupleType, SmallVectorImpl<Type> &types) {
           tupleType.getFlattenedTypes(types);
           return success();
         });
 
     decomposer.addDecomposeValueConversion([](OpBuilder &builder, Location loc,
                                               TupleType resultType, Value value,
                                               SmallVectorImpl<Value> &values) {
       for (unsigned i = 0, e = resultType.size(); i < e; ++i) {
         Value res = builder.create<test::GetTupleElementOp>(
             loc, resultType.getType(i), value, builder.getI32IntegerAttr(i));
         values.push_back(res);
       }
       return success();
     });
 
     typeConverter.addArgumentMaterialization(
         [](OpBuilder &builder, TupleType resultType, ValueRange inputs,
            Location loc) -> Optional<Value> {
           if (inputs.size() == 1)
             return llvm::None;
           TupleType tuple = builder.getTupleType(inputs.getTypes());
           Value value = builder.create<test::MakeTupleOp>(loc, tuple, inputs);
           return value;
         });
 
     populateDecomposeCallGraphTypesPatterns(context, typeConverter, decomposer,
                                             patterns);
 
     if (failed(applyPartialConversion(module, target, std::move(patterns))))
       return signalPassFailure();
   }
 };
 
 } // end anonymous namespace
 
 namespace mlir {
 namespace test {
 void registerTestDecomposeCallGraphTypes() {
   PassRegistration<TestDecomposeCallGraphTypes> pass(
       "test-decompose-call-graph-types",
       "Decomposes types at call graph boundaries.");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/Dialect/Vector/CMakeLists.txt

+# Exclude tests from libMLIR.so
+add_mlir_library(MLIRVectorTestPasses
+  TestVectorTransforms.cpp
+
+  EXCLUDE_FROM_LIBMLIR
+
+  LINK_LIBS PUBLIC
+  MLIRAffine
+  MLIRAnalysis
+  MLIRLinalg
+  MLIRMemRef
+  MLIRPass
+  MLIRSCF
+  MLIRStandard
+  MLIRTransformUtils
+  MLIRVector
+  MLIRVectorToSCF
+  )

mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp

 //===- TestVectorToVectorConversion.cpp - Test VectorTransfers lowering ---===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include <type_traits>
 
 #include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Conversion/VectorToSCF/ProgressiveVectorToSCF.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Linalg/IR/LinalgOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/SCF.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/Dialect/Vector/VectorTransforms.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
 using namespace mlir::vector;
 namespace {
 
 struct TestVectorToVectorConversion
     : public PassWrapper<TestVectorToVectorConversion, FunctionPass> {
   TestVectorToVectorConversion() = default;
   TestVectorToVectorConversion(const TestVectorToVectorConversion &pass) {}
 
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<AffineDialect>();
   }
 
   Option<bool> unroll{*this, "unroll", llvm::cl::desc("Include unrolling"),
                       llvm::cl::init(false)};
 
   void runOnFunction() override {
     auto *ctx = &getContext();
     RewritePatternSet patterns(ctx);
     if (unroll) {
       patterns.add<UnrollVectorPattern>(
           ctx,
           UnrollVectorOptions().setNativeShapeFn(getShape).setFilterConstraint(
               filter));
     }
     populateVectorToVectorCanonicalizationPatterns(patterns);
     populateVectorToVectorTransformationPatterns(patterns);
     populateBubbleVectorBitCastOpPatterns(patterns);
     populateCastAwayVectorLeadingOneDimPatterns(patterns);
     populateSplitVectorTransferPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 
 private:
   // Return the target shape based on op type.
   static Optional<SmallVector<int64_t, 4>> getShape(Operation *op) {
     if (isa<AddFOp, SelectOp, CmpFOp>(op))
       return SmallVector<int64_t, 4>(2, 2);
     if (isa<vector::ContractionOp>(op))
       return SmallVector<int64_t, 4>(3, 2);
     return llvm::None;
   }
 
   static LogicalResult filter(Operation *op) {
     return success(isa<AddFOp, SelectOp, CmpFOp, ContractionOp>(op));
   }
 };
 
 struct TestVectorSlicesConversion
     : public PassWrapper<TestVectorSlicesConversion, FunctionPass> {
   void runOnFunction() override {
     RewritePatternSet patterns(&getContext());
     populateVectorSlicesLoweringPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
 
 struct TestVectorContractionConversion
     : public PassWrapper<TestVectorContractionConversion, FunctionPass> {
   TestVectorContractionConversion() = default;
   TestVectorContractionConversion(const TestVectorContractionConversion &pass) {
   }
 
   Option<bool> lowerToFlatMatrix{
       *this, "vector-lower-matrix-intrinsics",
       llvm::cl::desc("Lower vector.contract to llvm.intr.matrix.multiply"),
       llvm::cl::init(false)};
   Option<bool> lowerToFlatTranspose{
       *this, "vector-flat-transpose",
       llvm::cl::desc("Lower 2-D vector.transpose to vector.flat_transpose"),
       llvm::cl::init(false)};
   Option<bool> lowerToOuterProduct{
       *this, "vector-outerproduct",
       llvm::cl::desc("Lower vector.contract to vector.outerproduct"),
       llvm::cl::init(false)};
   Option<bool> lowerToFilterOuterProduct{
       *this, "vector-filter-outerproduct",
       llvm::cl::desc("Lower vector.contract to vector.outerproduct but not for "
                      "vectors of size 4."),
       llvm::cl::init(false)};
 
   void runOnFunction() override {
     RewritePatternSet patterns(&getContext());
 
     // Test on one pattern in isolation.
     if (lowerToOuterProduct) {
       VectorContractLowering lowering = VectorContractLowering::OuterProduct;
       VectorTransformsOptions options{lowering};
       patterns.add<ContractionOpToOuterProductOpLowering>(options,
                                                           &getContext());
       (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
       return;
     }
 
     // Test on one pattern in isolation.
     if (lowerToFilterOuterProduct) {
       VectorContractLowering lowering = VectorContractLowering::OuterProduct;
       VectorTransformsOptions options{lowering};
       patterns.add<ContractionOpToOuterProductOpLowering>(
           options, &getContext(), [](vector::ContractionOp op) {
             // Only lowers vector.contract where the lhs as a type vector<MxNx?>
             // where M is not 4.
             if (op.getRhsType().getShape()[0] == 4)
               return failure();
             return success();
           });
       (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
       return;
     }
 
     // Test on all contract lowering patterns.
     VectorContractLowering contractLowering = VectorContractLowering::Dot;
     if (lowerToFlatMatrix)
       contractLowering = VectorContractLowering::Matmul;
     VectorTransposeLowering transposeLowering =
         VectorTransposeLowering::EltWise;
     if (lowerToFlatTranspose)
       transposeLowering = VectorTransposeLowering::Flat;
     VectorTransformsOptions options{contractLowering, transposeLowering};
     populateVectorContractLoweringPatterns(patterns, options);
     populateVectorTransposeLoweringPatterns(patterns, options);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
 
 struct TestVectorUnrollingPatterns
     : public PassWrapper<TestVectorUnrollingPatterns, FunctionPass> {
   TestVectorUnrollingPatterns() = default;
   TestVectorUnrollingPatterns(const TestVectorUnrollingPatterns &pass) {}
   void runOnFunction() override {
     MLIRContext *ctx = &getContext();
     RewritePatternSet patterns(ctx);
     patterns.add<UnrollVectorPattern>(
         ctx, UnrollVectorOptions()
                  .setNativeShape(ArrayRef<int64_t>{2, 2})
                  .setFilterConstraint([](Operation *op) {
                    return success(isa<AddFOp, vector::FMAOp>(op));
                  }));
 
     if (unrollBasedOnType) {
       UnrollVectorOptions::NativeShapeFnType nativeShapeFn =
           [](Operation *op) -> Optional<SmallVector<int64_t, 4>> {
         vector::ContractionOp contractOp = cast<vector::ContractionOp>(op);
         SmallVector<int64_t, 4> nativeShape = {4, 4, 2};
         if (auto floatType = contractOp.getLhsType()
                                  .getElementType()
                                  .dyn_cast<FloatType>()) {
           if (floatType.getWidth() == 16) {
             nativeShape[2] = 4;
           }
         }
         return nativeShape;
       };
       patterns.add<UnrollVectorPattern>(
           ctx, UnrollVectorOptions()
                    .setNativeShapeFn(nativeShapeFn)
                    .setFilterConstraint([](Operation *op) {
                      return success(isa<ContractionOp>(op));
                    }));
     } else {
       patterns.add<UnrollVectorPattern>(
           ctx, UnrollVectorOptions()
                    .setNativeShape(ArrayRef<int64_t>{2, 2, 2})
                    .setFilterConstraint([](Operation *op) {
                      return success(isa<ContractionOp>(op));
                    }));
     }
     populateVectorToVectorCanonicalizationPatterns(patterns);
     populateVectorToVectorTransformationPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 
   Option<bool> unrollBasedOnType{
       *this, "unroll-based-on-type",
       llvm::cl::desc("Set the unroll factor based on type of the operation"),
       llvm::cl::init(false)};
 };
 
 struct TestVectorDistributePatterns
     : public PassWrapper<TestVectorDistributePatterns, FunctionPass> {
   TestVectorDistributePatterns() = default;
   TestVectorDistributePatterns(const TestVectorDistributePatterns &pass) {}
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<VectorDialect>();
     registry.insert<AffineDialect>();
   }
   ListOption<int32_t> multiplicity{
       *this, "distribution-multiplicity", llvm::cl::MiscFlags::CommaSeparated,
       llvm::cl::desc("Set the multiplicity used for distributing vector")};
 
   void runOnFunction() override {
     MLIRContext *ctx = &getContext();
     RewritePatternSet patterns(ctx);
     FuncOp func = getFunction();
     func.walk([&](AddFOp op) {
       OpBuilder builder(op);
       if (auto vecType = op.getType().dyn_cast<VectorType>()) {
         SmallVector<int64_t, 2> mul;
         SmallVector<AffineExpr, 2> perm;
         SmallVector<Value, 2> ids;
         unsigned count = 0;
         // Remove the multiplicity of 1 and calculate the affine map based on
         // the multiplicity.
         SmallVector<int32_t, 4> m(multiplicity.begin(), multiplicity.end());
         for (unsigned i = 0, e = vecType.getRank(); i < e; i++) {
           if (i < m.size() && m[i] != 1 && vecType.getDimSize(i) % m[i] == 0) {
             mul.push_back(m[i]);
             ids.push_back(func.getArgument(count++));
             perm.push_back(getAffineDimExpr(i, ctx));
           }
         }
         auto map = AffineMap::get(op.getType().cast<VectorType>().getRank(), 0,
                                   perm, ctx);
         Optional<mlir::vector::DistributeOps> ops = distributPointwiseVectorOp(
             builder, op.getOperation(), ids, mul, map);
         if (ops.hasValue()) {
           SmallPtrSet<Operation *, 1> extractOp({ops->extract, ops->insert});
           op.getResult().replaceAllUsesExcept(ops->insert.getResult(),
                                               extractOp);
         }
       }
     });
     patterns.add<PointwiseExtractPattern>(ctx);
     populateVectorToVectorTransformationPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
 
 struct TestVectorToLoopPatterns
     : public PassWrapper<TestVectorToLoopPatterns, FunctionPass> {
   TestVectorToLoopPatterns() = default;
   TestVectorToLoopPatterns(const TestVectorToLoopPatterns &pass) {}
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<VectorDialect>();
     registry.insert<AffineDialect>();
   }
   Option<int32_t> multiplicity{
       *this, "distribution-multiplicity",
       llvm::cl::desc("Set the multiplicity used for distributing vector"),
       llvm::cl::init(32)};
   void runOnFunction() override {
     MLIRContext *ctx = &getContext();
     RewritePatternSet patterns(ctx);
     FuncOp func = getFunction();
     func.walk([&](AddFOp op) {
       // Check that the operation type can be broken down into a loop.
       VectorType type = op.getType().dyn_cast<VectorType>();
       if (!type || type.getRank() != 1 ||
           type.getNumElements() % multiplicity != 0)
         return mlir::WalkResult::advance();
       auto filterAlloc = [](Operation *op) {
         if (isa<ConstantOp, memref::AllocOp, CallOp>(op))
           return false;
         return true;
       };
       auto dependentOps = getSlice(op, filterAlloc);
       // Create a loop and move instructions from the Op slice into the loop.
       OpBuilder builder(op);
       auto zero = builder.create<ConstantOp>(
           op.getLoc(), builder.getIndexType(),
           builder.getIntegerAttr(builder.getIndexType(), 0));
       auto one = builder.create<ConstantOp>(
           op.getLoc(), builder.getIndexType(),
           builder.getIntegerAttr(builder.getIndexType(), 1));
       auto numIter = builder.create<ConstantOp>(
           op.getLoc(), builder.getIndexType(),
           builder.getIntegerAttr(builder.getIndexType(), multiplicity));
       auto forOp = builder.create<scf::ForOp>(op.getLoc(), zero, numIter, one);
       for (Operation *it : dependentOps) {
         it->moveBefore(forOp.getBody()->getTerminator());
       }
       auto map = AffineMap::getMultiDimIdentityMap(1, ctx);
       // break up the original op and let the patterns propagate.
       Optional<mlir::vector::DistributeOps> ops = distributPointwiseVectorOp(
           builder, op.getOperation(), {forOp.getInductionVar()}, {multiplicity},
           map);
       if (ops.hasValue()) {
         SmallPtrSet<Operation *, 1> extractOp({ops->extract, ops->insert});
         op.getResult().replaceAllUsesExcept(ops->insert.getResult(), extractOp);
       }
       return mlir::WalkResult::interrupt();
     });
     patterns.add<PointwiseExtractPattern>(ctx);
     populateVectorToVectorTransformationPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
 
 struct TestVectorTransferUnrollingPatterns
     : public PassWrapper<TestVectorTransferUnrollingPatterns, FunctionPass> {
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<AffineDialect>();
   }
   void runOnFunction() override {
     MLIRContext *ctx = &getContext();
     RewritePatternSet patterns(ctx);
     patterns.add<UnrollVectorPattern>(
         ctx,
         UnrollVectorOptions()
             .setNativeShape(ArrayRef<int64_t>{2, 2})
             .setFilterConstraint([](Operation *op) {
               return success(
                   isa<vector::TransferReadOp, vector::TransferWriteOp>(op));
             }));
     populateVectorToVectorCanonicalizationPatterns(patterns);
     populateVectorToVectorTransformationPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
 
 struct TestVectorTransferFullPartialSplitPatterns
     : public PassWrapper<TestVectorTransferFullPartialSplitPatterns,
                          FunctionPass> {
   TestVectorTransferFullPartialSplitPatterns() = default;
   TestVectorTransferFullPartialSplitPatterns(
       const TestVectorTransferFullPartialSplitPatterns &pass) {}
 
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<AffineDialect, linalg::LinalgDialect, memref::MemRefDialect,
                     scf::SCFDialect>();
   }
 
   Option<bool> useLinalgOps{
       *this, "use-linalg-copy",
       llvm::cl::desc("Split using a unmasked vector.transfer + linalg.fill + "
                      "linalg.copy operations."),
       llvm::cl::init(false)};
   void runOnFunction() override {
     MLIRContext *ctx = &getContext();
     RewritePatternSet patterns(ctx);
     VectorTransformsOptions options;
     if (useLinalgOps)
       options.setVectorTransferSplit(VectorTransferSplit::LinalgCopy);
     else
       options.setVectorTransferSplit(VectorTransferSplit::VectorTransfer);
     patterns.add<VectorTransferFullPartialRewriter>(ctx, options);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
 
 struct TestVectorTransferOpt
     : public PassWrapper<TestVectorTransferOpt, FunctionPass> {
   void runOnFunction() override { transferOpflowOpt(getFunction()); }
 };
 
 struct TestVectorTransferLoweringPatterns
     : public PassWrapper<TestVectorTransferLoweringPatterns, FunctionPass> {
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<memref::MemRefDialect>();
   }
   void runOnFunction() override {
     RewritePatternSet patterns(&getContext());
     populateVectorTransferLoweringPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
 
 struct TestVectorMultiReductionLoweringPatterns
     : public PassWrapper<TestVectorMultiReductionLoweringPatterns,
                          FunctionPass> {
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<memref::MemRefDialect>();
   }
   void runOnFunction() override {
     RewritePatternSet patterns(&getContext());
     populateVectorMultiReductionLoweringPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
 
 struct TestProgressiveVectorToSCFLoweringPatterns
     : public PassWrapper<TestProgressiveVectorToSCFLoweringPatterns,
                          FunctionPass> {
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<memref::MemRefDialect, scf::SCFDialect, AffineDialect>();
   }
   void runOnFunction() override {
     RewritePatternSet patterns(&getContext());
     populateProgressiveVectorToSCFConversionPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
 
 } // end anonymous namespace
 
 namespace mlir {
 namespace test {
 void registerTestVectorConversions() {
   PassRegistration<TestVectorToVectorConversion> vectorToVectorPass(
       "test-vector-to-vector-conversion",
       "Test conversion patterns between ops in the vector dialect");
 
   PassRegistration<TestVectorSlicesConversion> slicesPass(
       "test-vector-slices-conversion",
       "Test conversion patterns that lower slices ops in the vector dialect");
 
   PassRegistration<TestVectorContractionConversion> contractionPass(
       "test-vector-contraction-conversion",
       "Test conversion patterns that lower contract ops in the vector dialect");
 
   PassRegistration<TestVectorUnrollingPatterns> contractionUnrollingPass(
       "test-vector-unrolling-patterns",
       "Test conversion patterns to unroll contract ops in the vector dialect");
 
   PassRegistration<TestVectorTransferUnrollingPatterns> transferOpUnrollingPass(
       "test-vector-transfer-unrolling-patterns",
       "Test conversion patterns to unroll transfer ops in the vector dialect");
 
   PassRegistration<TestVectorTransferFullPartialSplitPatterns>
       vectorTransformFullPartialPass("test-vector-transfer-full-partial-split",
                                      "Test conversion patterns to split "
                                      "transfer ops via scf.if + linalg ops");
 
   PassRegistration<TestVectorDistributePatterns> distributePass(
       "test-vector-distribute-patterns",
       "Test conversion patterns to distribute vector ops in the vector "
       "dialect");
 
   PassRegistration<TestVectorToLoopPatterns> vectorToForLoop(
       "test-vector-to-forloop",
       "Test conversion patterns to break up a vector op into a for loop");
 
   PassRegistration<TestVectorTransferOpt> transferOpOpt(
       "test-vector-transferop-opt",
       "Test optimization transformations for transfer ops");
 
   PassRegistration<TestVectorTransferLoweringPatterns> transferOpLoweringPass(
       "test-vector-transfer-lowering-patterns",
       "Test conversion patterns to lower transfer ops to other vector ops");
 
   PassRegistration<TestProgressiveVectorToSCFLoweringPatterns> transferOpToSCF(
       "test-progressive-convert-vector-to-scf",
       "Test conversion patterns to progressively lower transfer ops to SCF");
 
   PassRegistration<TestVectorMultiReductionLoweringPatterns>
       multiDimReductionOpLoweringPass(
           "test-vector-multi-reduction-lowering-patterns",
           "Test conversion patterns to lower vector.multi_reduction to other "
           "vector ops");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/IR/CMakeLists.txt

 # Exclude tests from libMLIR.so
 add_mlir_library(MLIRTestIR
+  TestDominance.cpp
   TestFunc.cpp
   TestInterfaces.cpp
   TestMatchers.cpp
+  TestOpaqueLoc.cpp
   TestPrintDefUse.cpp
   TestPrintNesting.cpp
   TestSideEffects.cpp
   TestSlicing.cpp
   TestSymbolUses.cpp
   TestTypes.cpp
   TestVisitors.cpp
 
   EXCLUDE_FROM_LIBMLIR
 
   LINK_LIBS PUBLIC
   MLIRPass
   MLIRTestDialect
   )
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../Dialect/Test)
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/../Dialect/Test)

mlir/test/lib/IR/TestDominance.cpp

 //===- TestDominance.cpp - Test dominance construction and information
 //-------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains test passes for constructing and resolving dominance
 // information.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/IR/Dominance.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 
 /// Overloaded helper to call the right function based on whether we are testing
 /// dominance or post-dominance.
 static bool dominatesOrPostDominates(DominanceInfo &dominanceInfo, Block *a,
                                      Block *b) {
   return dominanceInfo.dominates(a, b);
 }
 
 static bool dominatesOrPostDominates(PostDominanceInfo &dominanceInfo, Block *a,
                                      Block *b) {
   return dominanceInfo.postDominates(a, b);
 }
 
 namespace {
 
 /// Helper class to print dominance information.
 class DominanceTest {
 public:
   /// Constructs a new test instance using the given operation.
   DominanceTest(Operation *operation) : operation(operation) {
     // Create unique ids for each block.
     operation->walk([&](Operation *nested) {
       if (blockIds.count(nested->getBlock()) > 0)
         return;
       blockIds.insert({nested->getBlock(), blockIds.size()});
     });
   }
 
   /// Prints dominance information of all blocks.
   template <typename DominanceT>
   void printDominance(DominanceT &dominanceInfo,
                       bool printCommonDominatorInfo) {
     DenseSet<Block *> parentVisited;
     operation->walk([&](Operation *op) {
       Block *block = op->getBlock();
       if (!parentVisited.insert(block).second)
         return;
 
       DenseSet<Block *> visited;
       operation->walk([&](Operation *nested) {
         Block *nestedBlock = nested->getBlock();
         if (!visited.insert(nestedBlock).second)
           return;
         if (printCommonDominatorInfo) {
           llvm::errs() << "Nearest(" << blockIds[block] << ", "
                        << blockIds[nestedBlock] << ") = ";
           Block *dom =
               dominanceInfo.findNearestCommonDominator(block, nestedBlock);
           if (dom)
             llvm::errs() << blockIds[dom];
           else
             llvm::errs() << "<no dom>";
           llvm::errs() << "\n";
         } else {
           if (std::is_same<DominanceInfo, DominanceT>::value)
             llvm::errs() << "dominates(";
           else
             llvm::errs() << "postdominates(";
           llvm::errs() << blockIds[block] << ", " << blockIds[nestedBlock]
                        << ") = ";
           if (dominatesOrPostDominates(dominanceInfo, block, nestedBlock))
             llvm::errs() << "true\n";
           else
             llvm::errs() << "false\n";
         }
       });
     });
   }
 
 private:
   Operation *operation;
   DenseMap<Block *, size_t> blockIds;
 };
 
 struct TestDominancePass : public PassWrapper<TestDominancePass, FunctionPass> {
 
   void runOnFunction() override {
     llvm::errs() << "Testing : " << getFunction().getName() << "\n";
     DominanceTest dominanceTest(getFunction());
 
     // Print dominance information.
     llvm::errs() << "--- DominanceInfo ---\n";
     dominanceTest.printDominance(getAnalysis<DominanceInfo>(),
                                  /*printCommonDominatorInfo=*/true);
 
     llvm::errs() << "--- PostDominanceInfo ---\n";
     dominanceTest.printDominance(getAnalysis<PostDominanceInfo>(),
                                  /*printCommonDominatorInfo=*/true);
 
     // Print dominance relationship between blocks.
     llvm::errs() << "--- Block Dominance relationship ---\n";
     dominanceTest.printDominance(getAnalysis<DominanceInfo>(),
                                  /*printCommonDominatorInfo=*/false);
 
     llvm::errs() << "--- Block PostDominance relationship ---\n";
     dominanceTest.printDominance(getAnalysis<PostDominanceInfo>(),
                                  /*printCommonDominatorInfo=*/false);
   }
 };
 
 } // end anonymous namespace
 
 namespace mlir {
 namespace test {
 void registerTestDominancePass() {
   PassRegistration<TestDominancePass>(
       "test-print-dominance",
       "Print the dominance information for multiple regions.");
 }
 } // namespace test
 } // namespace mlir

mlir/test/lib/IR/TestOpaqueLoc.cpp

 //===- TestOpaqueLoc.cpp - Pass to test opaque locations ------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 
 namespace {
 /// Pass that changes locations to opaque locations for each operation.
 /// It also takes all operations that are not function operations or

mlir/test/lib/Pass/CMakeLists.txt

 # Exclude tests from libMLIR.so
 add_mlir_library(MLIRTestPass
+  TestDynamicPipeline.cpp
   TestPassManager.cpp
 
   EXCLUDE_FROM_LIBMLIR
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Pass
 
   LINK_LIBS PUBLIC
   MLIRIR
   MLIRPass
   )

mlir/test/lib/Pass/TestDynamicPipeline.cpp

 //===------ TestDynamicPipeline.cpp --- dynamic pipeline test pass --------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements a pass to test the dynamic pipeline feature.
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/SCF/SCF.h"
-#include "mlir/IR/Builders.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
-#include "mlir/Transforms/LoopUtils.h"
-#include "mlir/Transforms/Passes.h"
 
 using namespace mlir;
 
 namespace {
 
 class TestDynamicPipelinePass
     : public PassWrapper<TestDynamicPipelinePass, OperationPass<>> {
 public:

mlir/test/lib/Transforms/CMakeLists.txt

 # Exclude tests from libMLIR.so
 add_mlir_library(MLIRTestTransforms
-  TestAffineLoopParametricTiling.cpp
-  TestExpandTanh.cpp
-  TestCallGraph.cpp
-  TestDecomposeCallGraphTypes.cpp
   TestConstantFold.cpp
-  TestConvVectorization.cpp
-  TestConvertCallOp.cpp
-  TestConvertGPUKernelToCubin.cpp
-  TestConvertGPUKernelToHsaco.cpp
-  TestDataLayoutQuery.cpp
-  TestDominance.cpp
-  TestDynamicPipeline.cpp
-  TestComposeSubView.cpp
-  TestLoopFusion.cpp
-  TestGpuMemoryPromotion.cpp
-  TestGpuParallelLoopMapping.cpp
-  TestGpuRewrite.cpp
   TestInlining.cpp
-  TestLinalgCodegenStrategy.cpp
-  TestLinalgElementwiseFusion.cpp
-  TestLinalgFusionTransforms.cpp
-  TestLinalgHoisting.cpp
-  TestLinalgTransforms.cpp
-  TestLiveness.cpp
+  TestLoopFusion.cpp
   TestLoopMapping.cpp
   TestLoopParametricTiling.cpp
   TestLoopUnrolling.cpp
-  TestNumberOfExecutions.cpp
-  TestOpaqueLoc.cpp
-  TestPolynomialApproximation.cpp
-  TestMemRefBoundCheck.cpp
-  TestMemRefDependenceCheck.cpp
-  TestMemRefStrideCalculation.cpp
-  TestSCFUtils.cpp
-  TestVectorTransforms.cpp
 
   EXCLUDE_FROM_LIBMLIR
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Transforms
 
-  DEPENDS
-  MLIRStandardOpsIncGen
-
   LINK_LIBS PUBLIC
-  MLIRAffine
-  MLIRAnalysis
-  MLIREDSC
-  MLIRGPU
-  MLIRGPU
-  MLIRGPUToGPURuntimeTransforms
-  MLIRLinalg
-  MLIRLinalgTransforms
-  MLIRLLVMToLLVMIRTranslation
-  MLIRMathTransforms
-  MLIRNVVMIR
-  MLIRNVVMToLLVMIRTranslation
-  MLIRPass
-  MLIRROCDLIR
-  MLIRROCDLToLLVMIRTranslation
-  MLIRSCF
-  MLIRSCFTransforms
-  MLIRStandardOpsTransforms
-  MLIRSparseTensor
-  MLIRSparseTensorTransforms
-  MLIRTargetLLVMIRExport
-  MLIRTestDialect
-  MLIRTransformUtils
-  MLIRVector
-  MLIRVectorToSCF
+  MLIRTransforms
   )
 
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../Dialect/Test)
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/../Dialect/Test)

mlir/test/lib/Transforms/TestConstantFold.cpp

 //===- TestConstantFold.cpp - Pass to test constant folding ---------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/StandardOps/IR/Ops.h"
-#include "mlir/IR/Builders.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/FoldUtils.h"
 #include "mlir/Transforms/Passes.h"
 #include "mlir/Transforms/Utils.h"
 
 using namespace mlir;
 
 namespace {

mlir/tools/mlir-opt/CMakeLists.txt

 set(LLVM_OPTIONAL_SOURCES
   null.cpp
 )
 
 get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
 set(LLVM_LINK_COMPONENTS
   Core
   Support
   AsmParser
   )
 
 if(MLIR_INCLUDE_TESTS)
   set(test_libs
+    MLIRTestStandardToLLVM
     MLIRAffineTransformsTestPasses
+    MLIRDLTITestPasses
+    MLIRGPUTestPasses
+    MLIRLinalgTestPasses
+    MLIRMathTestPasses
+    MLIRSCFTestPasses
     MLIRShapeTestPasses
     MLIRSPIRVTestPasses
+    MLIRStandardOpsTestPasses
+    MLIRVectorTestPasses
     MLIRTestAnalysis
     MLIRTestDialect
     MLIRTestIR
     MLIRTestPass
     MLIRTestReducer
     MLIRTestRewrite
     MLIRTestTransforms
     )