diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -15,6 +15,7 @@
 #define LLVM_IR_MODULE_H
 
 #include "llvm-c/Types.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringMap.h"
@@ -192,6 +193,8 @@
   NamedMDSymTabType NamedMDSymTab;  ///< NamedMDNode names.
   DataLayout DL;                  ///< DataLayout associated with the module
 
+  Optional<DenseMap<unsigned, unsigned>> IntrinsicsInModule = None;
+
   friend class Constant;
 
 /// @}
@@ -382,9 +385,12 @@
   /// exist, return null.
   Function *getFunction(StringRef Name) const;
 
-/// @}
-/// @name Global Variable Accessors
-/// @{
+  bool containsIntrinsicDeclaration(unsigned ID);
+  void intrinsicDeclarationRemoved(unsigned ID);
+
+  /// @}
+  /// @name Global Variable Accessors
+  /// @{
 
   /// Look up the specified global variable in the module symbol table. If it
   /// does not exist, return null. If AllowInternal is set to true, this
diff --git a/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
--- a/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
@@ -809,7 +809,20 @@
   ModifiedDT = true;
 }
 
+bool shouldRunMaskedMemLowering(Function &F) {
+  Module *M = F.getParent();
+  return M->containsIntrinsicDeclaration(Intrinsic::masked_load) ||
+         M->containsIntrinsicDeclaration(Intrinsic::masked_store) ||
+         M->containsIntrinsicDeclaration(Intrinsic::masked_gather) ||
+         M->containsIntrinsicDeclaration(Intrinsic::masked_scatter) ||
+         M->containsIntrinsicDeclaration(Intrinsic::masked_expandload) ||
+         M->containsIntrinsicDeclaration(Intrinsic::masked_compressstore);
+}
+
 bool ScalarizeMaskedMemIntrin::runOnFunction(Function &F) {
+  if (!shouldRunMaskedMemLowering(F))
+    return false;
+
   bool EverMadeChange = false;
 
   TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -242,6 +242,8 @@
 }
 
 void Function::eraseFromParent() {
+  if (isIntrinsic())
+    getParent()->intrinsicDeclarationRemoved(getIntrinsicID());
   getParent()->getFunctionList().erase(getIterator());
 }
 
diff --git a/llvm/lib/IR/Module.cpp b/llvm/lib/IR/Module.cpp
--- a/llvm/lib/IR/Module.cpp
+++ b/llvm/lib/IR/Module.cpp
@@ -149,6 +149,13 @@
                                      DL.getProgramAddressSpace(), Name);
     if (!New->isIntrinsic())       // Intrinsics get attrs set on construction
       New->setAttributes(AttributeList);
+    else {
+      if (IntrinsicsInModule) {
+        auto I = IntrinsicsInModule->insert({New->getIntrinsicID(), 0});
+        I.first->second++;
+      }
+    }
+
     FunctionList.push_back(New);
     return {Ty, New}; // Return the new prototype.
   }
@@ -174,6 +181,28 @@
   return dyn_cast_or_null<Function>(getNamedValue(Name));
 }
 
+bool Module::containsIntrinsicDeclaration(unsigned ID) {
+  if (!IntrinsicsInModule) {
+    DenseMap<unsigned, unsigned> Map;
+    IntrinsicsInModule = {Map};
+    for (const Function &Fn : FunctionList)
+      if (Fn.isIntrinsic()) {
+        auto I = IntrinsicsInModule->insert({Fn.getIntrinsicID(), 0});
+        I.first->second++;
+      }
+  }
+  return IntrinsicsInModule->find(ID) != IntrinsicsInModule->end();
+}
+
+void Module::intrinsicDeclarationRemoved(unsigned ID) {
+  if (!IntrinsicsInModule)
+    return;
+  auto I = IntrinsicsInModule->find(ID);
+  assert(I != IntrinsicsInModule->end());
+  if (--I->second)
+    IntrinsicsInModule->erase(ID);
+}
+
 //===----------------------------------------------------------------------===//
 // Methods for easy access to the global variables in the module.
 //
diff --git a/llvm/lib/Transforms/Scalar/LowerConstantIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerConstantIntrinsics.cpp
--- a/llvm/lib/Transforms/Scalar/LowerConstantIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerConstantIntrinsics.cpp
@@ -133,8 +133,17 @@
   return !Worklist.empty();
 }
 
+static bool shouldRunConstantLowering(Function &F) {
+  Module *M = F.getParent();
+  return M->containsIntrinsicDeclaration(Intrinsic::is_constant) ||
+         M->containsIntrinsicDeclaration(Intrinsic::objectsize);
+}
+
 PreservedAnalyses
 LowerConstantIntrinsicsPass::run(Function &F, FunctionAnalysisManager &AM) {
+  if (!shouldRunConstantLowering(F))
+    return PreservedAnalyses::all();
+
   if (lowerConstantIntrinsics(F, AM.getCachedResult<TargetLibraryAnalysis>(F)))
     return PreservedAnalyses::none();
 
@@ -155,6 +164,9 @@
   }
 
   bool runOnFunction(Function &F) override {
+    if (!shouldRunConstantLowering(F))
+      return false;
+
     auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
     const TargetLibraryInfo *TLI = TLIP ? &TLIP->getTLI(F) : nullptr;
     return lowerConstantIntrinsics(F, TLI);
diff --git a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
--- a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
@@ -363,8 +363,16 @@
   return Changed;
 }
 
+static bool shouldRunExpectLowering(Function &F) {
+  Module *M = F.getParent();
+  return M->containsIntrinsicDeclaration(Intrinsic::expect);
+}
+
 PreservedAnalyses LowerExpectIntrinsicPass::run(Function &F,
                                                 FunctionAnalysisManager &) {
+  if (!shouldRunExpectLowering(F))
+    return PreservedAnalyses::all();
+
   if (lowerExpectIntrinsic(F))
     return PreservedAnalyses::none();
 
@@ -385,7 +393,12 @@
     initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry());
   }
 
-  bool runOnFunction(Function &F) override { return lowerExpectIntrinsic(F); }
+  bool runOnFunction(Function &F) override {
+    if (!shouldRunExpectLowering(F))
+      return false;
+
+    return lowerExpectIntrinsic(F);
+  }
 };
 }