diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LostDebugLocObserver.h b/llvm/include/llvm/CodeGen/GlobalISel/LostDebugLocObserver.h
--- a/llvm/include/llvm/CodeGen/GlobalISel/LostDebugLocObserver.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LostDebugLocObserver.h
@@ -16,16 +16,17 @@
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 
 namespace llvm {
-class LostDebugLocObserver : public GISelChangeObserver {
+class LostDebugLocObserver : public GISelChangeObserver{
   StringRef DebugType;
   SmallSet<DebugLoc, 4> LostDebugLocs;
   SmallPtrSet<MachineInstr *, 4> PotentialMIsForDebugLocs;
   unsigned NumLostDebugLocs = 0;
 
 public:
+  LostDebugLocObserver() = default;
   LostDebugLocObserver(StringRef DebugType) : DebugType(DebugType) {}
 
-  unsigned getNumLostDebugLocs() const { return NumLostDebugLocs; }
+  virtual unsigned getNumLostDebugLocs() const { return NumLostDebugLocs; }
 
   /// Call this to indicate that it's a good point to assess whether locations
   /// have been lost. Typically this will be when a logical change has been
@@ -35,16 +36,31 @@
   /// CheckDebugLocs is false, it will just reset ready for the next checkpoint
   /// without checking anything. This can be helpful to limit the detection to
   /// easy-to-fix portions of an algorithm before allowing more difficult ones.
-  void checkpoint(bool CheckDebugLocs = true);
+  virtual void checkpoint(bool CheckDebugLocs = true);
 
-  void createdInstr(MachineInstr &MI) override;
-  void erasingInstr(MachineInstr &MI) override;
-  void changingInstr(MachineInstr &MI) override;
-  void changedInstr(MachineInstr &MI) override;
+  virtual void createdInstr(MachineInstr &MI) override;
+  virtual void erasingInstr(MachineInstr &MI) override;
+  virtual void changingInstr(MachineInstr &MI) override;
+  virtual void changedInstr(MachineInstr &MI) override;
 
 private:
   void analyzeDebugLocations();
 };
 
+/// This is an empty observer that can be installed when assertions are
+/// disabled, to save compile time without requiring more intrusive ifdefs at
+/// the user call sites.
+class NullLostDebugLocObserver : public LostDebugLocObserver {
+public:
+  NullLostDebugLocObserver() = default;
+  virtual unsigned getNumLostDebugLocs() const override { return 0; }
+  virtual void checkpoint(bool CheckDebugLocs = true) override {}
+
+  virtual void createdInstr(MachineInstr &MI) override {}
+  virtual void erasingInstr(MachineInstr &MI) override {}
+  virtual void changingInstr(MachineInstr &MI) override {}
+  virtual void changedInstr(MachineInstr &MI) override {}
+};
+
 } // namespace llvm
 #endif // LLVM_CODEGEN_GLOBALISEL_LOSTDEBUGLOCOBSERVER_H
diff --git a/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp b/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp
--- a/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp
@@ -339,7 +339,11 @@
     AuxObservers.push_back(CSEInfo);
   }
   assert(!CSEInfo || !errorToBool(CSEInfo->verify()));
+#ifndef NDEBUG
   LostDebugLocObserver LocObserver(DEBUG_TYPE);
+#else
+  NullLostDebugLocObserver LocObserver;
+#endif
   if (VerifyDebugLocs > DebugLocVerifyLevel::None)
     AuxObservers.push_back(&LocObserver);