diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp
--- a/mlir/lib/IR/Operation.cpp
+++ b/mlir/lib/IR/Operation.cpp
@@ -220,13 +220,22 @@
 // Diagnostics
 //===----------------------------------------------------------------------===//
 
+/// Returns true if the op has one or more non-empty regions. This is meant to
+/// be a proxy for whether `op` will be printed on multiple lines.
+static bool hasNonEmptyRegions(Operation *op) {
+  return llvm::any_of(op->getRegions(),
+                      [](Region &region) { return !region.empty(); });
+}
+
 /// Emit an error about fatal conditions with this operation, reporting up to
 /// any diagnostic handlers that may be listening.
 InFlightDiagnostic Operation::emitError(const Twine &message) {
   InFlightDiagnostic diag = mlir::emitError(getLoc(), message);
   if (getContext()->shouldPrintOpOnDiagnostic()) {
+    // Emit the op on a new line if the op has one or more non-empty regions.
     diag.attachNote(getLoc())
-        .append("see current operation: ")
+        .append("see current operation:")
+        .append(hasNonEmptyRegions(this) ? '\n' : ' ')
         .appendOp(*this, OpPrintingFlags().printGenericOpForm());
   }
   return diag;
@@ -237,7 +246,11 @@
 InFlightDiagnostic Operation::emitWarning(const Twine &message) {
   InFlightDiagnostic diag = mlir::emitWarning(getLoc(), message);
   if (getContext()->shouldPrintOpOnDiagnostic())
-    diag.attachNote(getLoc()) << "see current operation: " << *this;
+    // Emit the op on a new line if the op has one or more non-empty regions.
+    diag.attachNote(getLoc())
+            .append("see current operation:")
+            .append(hasNonEmptyRegions(this) ? '\n' : ' ')
+        << *this;
   return diag;
 }
 
@@ -246,7 +259,11 @@
 InFlightDiagnostic Operation::emitRemark(const Twine &message) {
   InFlightDiagnostic diag = mlir::emitRemark(getLoc(), message);
   if (getContext()->shouldPrintOpOnDiagnostic())
-    diag.attachNote(getLoc()) << "see current operation: " << *this;
+    // Emit the op on a new line if the op has one or more non-empty regions.
+    diag.attachNote(getLoc())
+            .append("see current operation:")
+            .append(hasNonEmptyRegions(this) ? '\n' : ' ')
+        << *this;
   return diag;
 }
 
diff --git a/mlir/test/IR/print-op-on-diagnostic.mlir b/mlir/test/IR/print-op-on-diagnostic.mlir
--- a/mlir/test/IR/print-op-on-diagnostic.mlir
+++ b/mlir/test/IR/print-op-on-diagnostic.mlir
@@ -1,7 +1,18 @@
-// RUN: not mlir-opt %s -mlir-print-op-on-diagnostic 2>&1 | FileCheck %s
+// RUN: not mlir-opt -split-input-file %s -mlir-print-op-on-diagnostic 2>&1 | FileCheck %s
 
 // This file tests the functionality of 'mlir-print-op-on-diagnostic'.
 
 // CHECK: {{invalid to use 'test.invalid_attr'}}
-// CHECK: {{see current operation: "builtin.module"()}}
+// CHECK: see current operation:
+// CHECK-NEXT: "builtin.module"()
 module attributes {test.invalid_attr} {}
+
+// -----
+
+func.func @foo() {
+  "test.foo"(%cst) : (index) -> ()
+  // CHECK: {{operand #0 does not dominate this use}}
+  // CHECK: {{see current operation: "test.foo"(.*)}}
+  %cst = arith.constant 0 : index
+  return
+}