diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -537,7 +537,9 @@
   case Instruction::Call:
   case Instruction::Invoke:
   case Instruction::CallBr:
-    return !cast<CallBase>(this)->doesNotReadMemory();
+    // A byval argument is read/copied at the call site.
+    return !cast<CallBase>(this)->doesNotReadMemory() ||
+           cast<CallBase>(this)->hasByValArgument();
   case Instruction::Store:
     return !cast<StoreInst>(this)->isUnordered();
   }
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -1687,11 +1687,6 @@
     }
     if (I.mayReadOrWriteMemory())
       FI.RWInsts.push_back(&I);
-    else if (CallBase *CB = dyn_cast<CallBase>(&I)) {
-      // A byval argument on a readnone function is still read at the call site.
-      if (CB->hasByValArgument())
-        FI.RWInsts.push_back(&I);
-    }
   }
 
   if (F.hasFnAttribute(Attribute::AlwaysInline) &&
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -5836,14 +5836,9 @@
       for (const Use &UserIUse : UserI->uses())
         Uses.insert(&UserIUse);
 
-    // If UserI might touch memory we analyze the use in detail. We also do that
-    // for readnone call sites when the use is a byval argument because the call
-    // site performs a copy of the memory.
+    // If UserI might touch memory we analyze the use in detail.
     if (UserI->mayReadOrWriteMemory())
       analyzeUseIn(A, U, UserI);
-    else if (CallBase *CB = dyn_cast<CallBase>(UserI))
-      if (CB->isByValArgument(CB->getArgOperandNo(U)))
-      analyzeUseIn(A, U, UserI);
   }
 
   return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED