diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -1174,8 +1174,10 @@
     ChangeStatus Changed = ChangeStatus::UNCHANGED;
     for (const auto &It : OtherAAImpl.getState()) {
       AA::OffsetAndSize OAS = AA::OffsetAndSize::getUnknown();
-      if (Offset != AA::OffsetAndSize::Unknown)
+      if (Offset != AA::OffsetAndSize::Unknown &&
+          !It.first.offsetOrSizeAreUnknown()) {
         OAS = AA::OffsetAndSize(It.first.Offset + Offset, It.first.Size);
+      }
       Accesses *Bin = AccessBins.lookup(OAS);
       for (const AAPointerInfo::Access &RAcc : *It.second) {
         if (IsByval && !RAcc.isRead())
diff --git a/llvm/test/Transforms/Attributor/call-simplify-pointer-info.ll b/llvm/test/Transforms/Attributor/call-simplify-pointer-info.ll
new file mode 100644
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/call-simplify-pointer-info.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-attributes --check-globals
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=TUNIT
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CGSCC
+;
+
+define internal i8 @read_arg(i8* %p) {
+; CGSCC: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; CGSCC-LABEL: define {{[^@]+}}@read_arg
+; CGSCC-SAME: (i8* nocapture nofree noundef nonnull readonly dereferenceable(1022) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CGSCC-NEXT:  entry:
+; CGSCC-NEXT:    [[L:%.*]] = load i8, i8* [[P]], align 1
+; CGSCC-NEXT:    ret i8 [[L]]
+;
+entry:
+  %l = load i8, i8* %p, align 1
+  ret i8 %l
+}
+
+define internal i8 @read_arg_index(i8* %p, i64 %index) {
+; TUNIT: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; TUNIT-LABEL: define {{[^@]+}}@read_arg_index
+; TUNIT-SAME: (i8* nocapture nofree noundef nonnull readonly align 2 dereferenceable(1022) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; TUNIT-NEXT:  entry:
+; TUNIT-NEXT:    [[L:%.*]] = load i8, i8* [[P]], align 2
+; TUNIT-NEXT:    ret i8 [[L]]
+;
+; CGSCC: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; CGSCC-LABEL: define {{[^@]+}}@read_arg_index
+; CGSCC-SAME: (i8* nocapture nofree noundef nonnull readonly dereferenceable(1022) [[P:%.*]]) #[[ATTR0]] {
+; CGSCC-NEXT:  entry:
+; CGSCC-NEXT:    [[L:%.*]] = load i8, i8* [[P]], align 1
+; CGSCC-NEXT:    ret i8 [[L]]
+;
+entry:
+  %g = getelementptr inbounds i8, i8* %p, i64 %index
+  %l = load i8, i8* %g, align 1
+  ret i8 %l
+}
+
+define i8 @call_simplifiable_1() {
+; TUNIT: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; TUNIT-LABEL: define {{[^@]+}}@call_simplifiable_1
+; TUNIT-SAME: () #[[ATTR1:[0-9]+]] {
+; TUNIT-NEXT:  entry:
+; TUNIT-NEXT:    [[BYTES:%.*]] = alloca [1024 x i8], align 16
+; TUNIT-NEXT:    [[I0:%.*]] = getelementptr inbounds [1024 x i8], [1024 x i8]* [[BYTES]], i64 0, i64 2
+; TUNIT-NEXT:    ret i8 2
+;
+; CGSCC: Function Attrs: nofree nosync nounwind readnone willreturn
+; CGSCC-LABEL: define {{[^@]+}}@call_simplifiable_1
+; CGSCC-SAME: () #[[ATTR1:[0-9]+]] {
+; CGSCC-NEXT:  entry:
+; CGSCC-NEXT:    [[BYTES:%.*]] = alloca [1024 x i8], align 16
+; CGSCC-NEXT:    [[I0:%.*]] = getelementptr inbounds [1024 x i8], [1024 x i8]* [[BYTES]], i64 0, i64 2
+; CGSCC-NEXT:    store i8 2, i8* [[I0]], align 2
+; CGSCC-NEXT:    [[R:%.*]] = call i8 @read_arg(i8* nocapture nofree noundef nonnull readonly align 2 dereferenceable(1022) [[I0]]) #[[ATTR2:[0-9]+]]
+; CGSCC-NEXT:    ret i8 [[R]]
+;
+entry:
+  %Bytes = alloca [1024 x i8], align 16
+  %i0 = getelementptr inbounds [1024 x i8], [1024 x i8]* %Bytes, i64 0, i64 2
+  store i8 2, i8* %i0, align 1
+  %r = call i8 @read_arg(i8* %i0)
+  ret i8 %r
+}
+
+define i8 @call_not_simplifiable_1() {
+; TUNIT: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; TUNIT-LABEL: define {{[^@]+}}@call_not_simplifiable_1
+; TUNIT-SAME: () #[[ATTR1]] {
+; TUNIT-NEXT:  entry:
+; TUNIT-NEXT:    [[BYTES:%.*]] = alloca [1024 x i8], align 16
+; TUNIT-NEXT:    [[I0:%.*]] = getelementptr inbounds [1024 x i8], [1024 x i8]* [[BYTES]], i64 0, i64 2
+; TUNIT-NEXT:    store i8 2, i8* [[I0]], align 2
+; TUNIT-NEXT:    [[R:%.*]] = call i8 @read_arg_index(i8* nocapture nofree noundef nonnull readonly align 2 dereferenceable(1022) [[I0]]) #[[ATTR2:[0-9]+]]
+; TUNIT-NEXT:    ret i8 [[R]]
+;
+; CGSCC: Function Attrs: nofree nosync nounwind readnone willreturn
+; CGSCC-LABEL: define {{[^@]+}}@call_not_simplifiable_1
+; CGSCC-SAME: () #[[ATTR1]] {
+; CGSCC-NEXT:  entry:
+; CGSCC-NEXT:    [[BYTES:%.*]] = alloca [1024 x i8], align 16
+; CGSCC-NEXT:    [[I0:%.*]] = getelementptr inbounds [1024 x i8], [1024 x i8]* [[BYTES]], i64 0, i64 2
+; CGSCC-NEXT:    store i8 2, i8* [[I0]], align 2
+; CGSCC-NEXT:    [[R:%.*]] = call i8 @read_arg_index(i8* nocapture nofree noundef nonnull readonly align 2 dereferenceable(1022) [[I0]]) #[[ATTR2]]
+; CGSCC-NEXT:    ret i8 [[R]]
+;
+entry:
+  %Bytes = alloca [1024 x i8], align 16
+  %i0 = getelementptr inbounds [1024 x i8], [1024 x i8]* %Bytes, i64 0, i64 2
+  store i8 2, i8* %i0, align 1
+  %r = call i8 @read_arg_index(i8* %i0, i64 0)
+  ret i8 %r
+}
+
+;.
+; TUNIT: attributes #[[ATTR0]] = { argmemonly nofree norecurse nosync nounwind readonly willreturn }
+; TUNIT: attributes #[[ATTR1]] = { nofree norecurse nosync nounwind readnone willreturn }
+; TUNIT: attributes #[[ATTR2]] = { nofree nosync nounwind readonly willreturn }
+;.
+; CGSCC: attributes #[[ATTR0]] = { argmemonly nofree norecurse nosync nounwind readonly willreturn }
+; CGSCC: attributes #[[ATTR1]] = { nofree nosync nounwind readnone willreturn }
+; CGSCC: attributes #[[ATTR2]] = { readonly willreturn }
+;.