diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -463,36 +463,6 @@
   }
 };
 
-/// Swap extract_slice(fill) to fill(extract_slice).
-///
-/// Only swap the two ops if the extract_slice is the only user of the fill.
-struct SwapExtractSliceOfFill : OpRewritePattern<tensor::ExtractSliceOp> {
-  using OpRewritePattern<tensor::ExtractSliceOp>::OpRewritePattern;
-  LogicalResult matchAndRewrite(tensor::ExtractSliceOp extractSliceOp,
-                                PatternRewriter &rewriter) const override {
-    auto oldFill = extractSliceOp.getSource().getDefiningOp<FillOp>();
-    if (!oldFill)
-      return failure();
-    // Only swap the ops if there is no other user of the fill.
-    if (!extractSliceOp.getSource().hasOneUse())
-      return failure();
-    // Extract from the old fill's source.
-    rewriter.updateRootInPlace(extractSliceOp, [&]() {
-      extractSliceOp.getSourceMutable().assign(oldFill.output());
-    });
-    // Create a new fill and remove the old one.
-    rewriter.setInsertionPointAfter(extractSliceOp);
-    auto newFill =
-        rewriter.create<FillOp>(oldFill.getLoc(), ValueRange{oldFill.value()},
-                                ValueRange{extractSliceOp.getResult()});
-    rewriter.eraseOp(oldFill);
-    // Use the new fill instead of the extract_slice.
-    rewriter.replaceAllUsesExcept(extractSliceOp.getResult(),
-                                  newFill.getResult(0), newFill);
-    return success();
-  }
-};
-
 /// Fold tensor.pad(linalg.fill) into linalg.fill if the padding value and the
 /// filling value are the same.
 struct FoldFillWithPad final : public OpRewritePattern<tensor::PadOp> {
@@ -637,7 +607,7 @@
   results
       .add<FoldFillWithPad, FoldFillWithTensorReshape<tensor::CollapseShapeOp>,
            FoldFillWithTensorReshape<tensor::ExpandShapeOp>,
-           FoldInsertPadIntoFill, SwapExtractSliceOfFill>(context);
+           FoldInsertPadIntoFill>(context);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/Linalg/canonicalize.mlir b/mlir/test/Dialect/Linalg/canonicalize.mlir
--- a/mlir/test/Dialect/Linalg/canonicalize.mlir
+++ b/mlir/test/Dialect/Linalg/canonicalize.mlir
@@ -312,20 +312,6 @@
 
 // -----
 
-// CHECK-LABEL: func @fold_fill_extract_slice(
-//  CHECK-SAME:     %[[t:.*]]: tensor<1x1xf32>
-func.func @fold_fill_extract_slice(%t: tensor<1x1xf32>) -> (tensor<f32>) {
-  %cst = arith.constant 0.000000e+00 : f32
-  // CHECK: %[[e:.*]] = tensor.extract_slice %[[t]]
-  // CHECK: %[[f:.*]] = linalg.fill {{.*}} outs(%[[e]] : tensor<f32>)
-  %0 = linalg.fill ins(%cst : f32) outs(%t : tensor<1x1xf32>) -> tensor<1x1xf32>
-  %1 = tensor.extract_slice %0[0, 0] [1, 1] [1, 1] : tensor<1x1xf32> to tensor<f32>
-  // CHECK: return %[[f]]
-  return %1 : tensor<f32>
-}
-
-// -----
-
 //       CHECK: func @fold_fill_reshape_dynamic
 //  CHECK-SAME:   %[[ARG0:.+]]: tensor<?x?x?x?x?xf32>
 func.func @fold_fill_reshape_dynamic(%arg0 : tensor<?x?x?x?x?xf32>) -> tensor<?x?xf32> {