Index: lib/Transforms/Utils/Local.cpp
===================================================================
--- lib/Transforms/Utils/Local.cpp
+++ lib/Transforms/Utils/Local.cpp
@@ -1426,7 +1426,8 @@
   // then we will just pick it from J - even when they are different.
   // FIXME: we should try to preserve both invariant.group md if they are
   // different, but right now instruction can only have one invariant.group.
-  if (auto *JMD = J->getMetadata(LLVMContext::MD_invariant_group))
+  if (auto *JMD = J->getMetadata(LLVMContext::MD_invariant_group) &&
+      isa<LoadInst>(K))
     K->setMetadata(LLVMContext::MD_invariant_group, JMD);
 
 }