diff --git a/mlir/lib/Target/LLVMIR/ConvertToLLVMIR.cpp b/mlir/lib/Target/LLVMIR/ConvertToLLVMIR.cpp
--- a/mlir/lib/Target/LLVMIR/ConvertToLLVMIR.cpp
+++ b/mlir/lib/Target/LLVMIR/ConvertToLLVMIR.cpp
@@ -17,6 +17,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/Verifier.h"
 #include "llvm/Support/ToolOutputFile.h"
 
 using namespace mlir;
@@ -24,7 +25,11 @@
 std::unique_ptr<llvm::Module>
 mlir::translateModuleToLLVMIR(ModuleOp m, llvm::LLVMContext &llvmContext,
                               StringRef name) {
-  return LLVM::ModuleTranslation::translateModule<>(m, llvmContext, name);
+  auto llvmModule =
+      LLVM::ModuleTranslation::translateModule<>(m, llvmContext, name);
+  if (verifyModule(*llvmModule))
+    emitError(m.getLoc(), "LLVM IR fails to verify");
+  return llvmModule;
 }
 
 namespace mlir {
diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
--- a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
@@ -71,6 +71,18 @@
   if (!compileUnit || !func.walk(interruptIfValidLocation).wasInterrupted())
     return;
 
+  // If we are to create debug info for the function, we need to ensure that all
+  // inlinable calls in it are with debug info, otherwise the LLVM verifier will
+  // complain. For now, be more restricted and treat all calls as inlinable.
+  const bool hasCallWithoutDebugInfo =
+      func.walk([](LLVM::CallOp call) {
+            return call.getLoc().isa<UnknownLoc>() ? WalkResult::interrupt()
+                                                   : WalkResult::advance();
+          })
+          .wasInterrupted();
+  if (hasCallWithoutDebugInfo)
+    return;
+
   FileLineColLoc fileLoc = extractFileLoc(func.getLoc());
   auto *file = translateFile(fileLoc ? fileLoc.getFilename() : "<unknown>");
   unsigned line = fileLoc ? fileLoc.getLine() : 0;
diff --git a/mlir/test/Target/llvmir-debug.mlir b/mlir/test/Target/llvmir-debug.mlir
--- a/mlir/test/Target/llvmir-debug.mlir
+++ b/mlir/test/Target/llvmir-debug.mlir
@@ -9,10 +9,6 @@
 // CHECK-LABEL: define void @func_with_debug()
 // CHECK-SAME: !dbg ![[FUNC_LOC:[0-9]+]]
 llvm.func @func_with_debug() {
-  // CHECK: call void @func_no_debug()
-  // CHECK-NOT: !dbg
-  llvm.call @func_no_debug() : () -> () loc(unknown)
-
   // CHECK: call void @func_no_debug(), !dbg ![[CALLSITE_LOC:[0-9]+]]
   llvm.call @func_no_debug() : () -> () loc(callsite("mysource.cc":3:4 at "mysource.cc":5:6))