diff --git a/llvm/lib/Analysis/AliasSetTracker.cpp b/llvm/lib/Analysis/AliasSetTracker.cpp
--- a/llvm/lib/Analysis/AliasSetTracker.cpp
+++ b/llvm/lib/Analysis/AliasSetTracker.cpp
@@ -233,8 +233,8 @@
   if (AliasAny)
     return true;
 
-  assert(Inst->mayReadOrWriteMemory() &&
-         "Instruction must either read or write memory.");
+  if (!Inst->mayReadOrWriteMemory())
+    return false;
 
   for (unsigned i = 0, e = UnknownInsts.size(); i != e; ++i) {
     if (auto *UnknownInst = getUnknownInst(i)) {
diff --git a/llvm/test/Transforms/LICM/pr51333.ll b/llvm/test/Transforms/LICM/pr51333.ll
new file mode 100644
--- /dev/null
+++ b/llvm/test/Transforms/LICM/pr51333.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -passes='early-cse<memssa>,loop-mssa(licm)' < %s | FileCheck %s
+
+; This used to assert because was have a MemoryDef for what turns out to be
+; a readnone call after EarlyCSE.
+
+@fn_ptr = external global void ()*, align 1
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[DO_BODY:%.*]]
+; CHECK:       do.body:
+; CHECK-NEXT:    store void ()* @readnone_fn, void ()** @fn_ptr, align 8
+; CHECK-NEXT:    call void @readnone_fn()
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[DO_BODY]]
+;
+entry:
+  br label %do.body
+
+do.body:
+  store void ()* @readnone_fn, void ()** @fn_ptr
+  %fn = load void ()*, void ()** @fn_ptr
+  call void %fn()
+  call void @foo()
+  br label %do.body
+}
+
+declare void @foo()
+declare void @readnone_fn() readnone