diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -562,12 +562,19 @@
 //===----------------------------------------------------------------------===//
 
 OpFoldResult ToTensorOp::fold(FoldAdaptor) {
-  if (auto toMemref = getMemref().getDefiningOp<ToMemrefOp>())
-    // Approximate alias analysis by conservatively folding only when no there
-    // is no interleaved operation.
+  if (auto toMemref = getMemref().getDefiningOp<ToMemrefOp>()) {
+    // Approximate alias analysis by conservatively folding only when know there
+    // is no interleaved user of the result memref.
+    bool canResultMemrefMutate =
+        llvm::any_of(toMemref->getUsers(), [&](Operation *op) {
+          if (op->getBlock() == this->getOperation()->getBlock())
+            return op->isBeforeInBlock(*this);
+          return true;
+        });
     if (toMemref->getBlock() == this->getOperation()->getBlock() &&
-        toMemref->getNextNode() == this->getOperation())
+        !canResultMemrefMutate)
       return toMemref.getTensor();
+  }
   return {};
 }
 
diff --git a/mlir/test/Dialect/Bufferization/canonicalize.mlir b/mlir/test/Dialect/Bufferization/canonicalize.mlir
--- a/mlir/test/Dialect/Bufferization/canonicalize.mlir
+++ b/mlir/test/Dialect/Bufferization/canonicalize.mlir
@@ -15,6 +15,38 @@
 
 // -----
 
+// Folding of to_tensor(to_memref(t)) -> t is allowed when the buffer is not mutated before the to_tensor.
+// CHECK-LABEL:   func.func @tensor_load_of_unmutated_buffer(
+func.func @tensor_load_of_unmutated_buffer(%arg0: tensor<?xf32>) -> (tensor<?xi32>, tensor<?xf32>) {
+  %0 = bufferization.to_memref %arg0 : memref<?xf32>
+  %1 = "tosa.cast"(%arg0) : (tensor<?xf32>) -> tensor<?xi32>
+  %2 = bufferization.to_tensor %0 : memref<?xf32>
+  return %1, %2 : tensor<?xi32>, tensor<?xf32>
+}
+
+// CHECK-SAME:   %[[arg0:.*]]: tensor<?xf32>) -> (tensor<?xi32>, tensor<?xf32>) {
+// CHECK: %[[cast:.*]] = "tosa.cast"(%[[arg0]]) : (tensor<?xf32>) -> tensor<?xi32>
+// CHECK: return %[[cast]], %[[arg0]] : tensor<?xi32>, tensor<?xf32>
+
+// -----
+
+// Folding of to_tensor(to_memref(t)) -> t is not allowed since t may mutate before the to_tensor.
+// CHECK-LABEL:   func.func @tensor_load_of_mutated_buffer(
+func.func @tensor_load_of_mutated_buffer(%arg0: tensor<?xf32>) -> (tensor<?xf32>) {
+  %0 = bufferization.to_memref %arg0 : memref<?xf32>
+  %1 = "use"(%0) : (memref<?xf32>) -> memref<?xf32>
+  %2 = bufferization.to_tensor %0 : memref<?xf32>
+  return %2 : tensor<?xf32>
+}
+
+// CHECK-SAME:  %[[VAL_0:.*]]: tensor<?xf32>) -> tensor<?xf32> {
+// CHECK: %[[VAL_1:.*]] = bufferization.to_memref %[[VAL_0]] : memref<?xf32>
+// CHECK: %[[VAL_2:.*]] = "use"(%[[VAL_1]]) : (memref<?xf32>) -> memref<?xf32>
+// CHECK: %[[VAL_3:.*]] = bufferization.to_tensor %[[VAL_1]] : memref<?xf32>
+// CHECK: return %[[VAL_3]] : tensor<?xf32>
+
+// -----
+
 // Basic folding of to_memref(to_tensor(m)) -> m
 // CHECK-LABEL: func @buffer_cast_of_tensor_load(
 func.func @buffer_cast_of_tensor_load(%arg0: memref<?xf32>) -> memref<?xf32> {