diff --git a/mlir/test/Dialect/Linalg/decompose-ops.mlir b/mlir/test/Dialect/Linalg/decompose-ops.mlir
--- a/mlir/test/Dialect/Linalg/decompose-ops.mlir
+++ b/mlir/test/Dialect/Linalg/decompose-ops.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt -test-linalg-decompose-ops -cse -split-input-file %s | FileCheck %s
-// RUN: mlir-opt -test-linalg-decompose-ops -cse -canonicalize -split-input-file %s | FileCheck %s --check-prefix=CANONICALIZECHECK
+// RUN: mlir-opt -pass-pipeline="test-linalg-decompose-ops{canonicalize-decomposed-ops=false}, cse" -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -pass-pipeline="test-linalg-decompose-ops, cse" -test-linalg-decompose-ops -cse -canonicalize -split-input-file %s | FileCheck %s --check-prefix=CANONICALIZECHECK
 
 func.func @simple_op(%arg0 : tensor<?x?xf32>, %arg1 : tensor<?xf32>, %arg2 : tensor<?xf32>)
     -> (tensor<?x?xf32>, tensor<?x?xf32>) {
diff --git a/mlir/test/lib/Dialect/Linalg/TestLinalgDecomposeOps.cpp b/mlir/test/lib/Dialect/Linalg/TestLinalgDecomposeOps.cpp
--- a/mlir/test/lib/Dialect/Linalg/TestLinalgDecomposeOps.cpp
+++ b/mlir/test/lib/Dialect/Linalg/TestLinalgDecomposeOps.cpp
@@ -33,10 +33,19 @@
     return "Test Linalg decomposition patterns";
   }
 
+  Option<bool> canonicalizeDecomposedOps{
+      *this, "canonicalize-decomposed-ops",
+      llvm::cl::desc("Canonicalize the decomposed ops."),
+      llvm::cl::init(false)};
+
   void runOnOperation() override {
     MLIRContext *context = &this->getContext();
     RewritePatternSet decompositionPatterns(context);
     linalg::populateDecomposeLinalgOpsPattern(decompositionPatterns);
+    if (canonicalizeDecomposedOps) {
+      linalg::GenericOp::getCanonicalizationPatterns(decompositionPatterns,
+                                                     context);
+    }
     if (failed(applyPatternsAndFoldGreedily(
             getOperation(), std::move(decompositionPatterns)))) {
       return signalPassFailure();