[DSE] Teach the pass about partial overwrite of atomic memory intrinsics

[dneilson]

Summary

This change teaches DSE that the atomic memory intrinsics can be overwriten
partially in the same way as the non-atomic forms. Specifically, that the
atomic memcpy & memset can be shortened at the end and that the atomic memset
can be shortened at the beginning, if they partially overwritten
by later stores.

[dneilson]

ping - any reviewers able to take a crack at this?

[mkazantsev]

I don't see any problems with the code (other than the nit I've pointed out), but I do not know this logic well enough to make the final judgement. Adding some people who contributed to DSE recently. Can you guys take a look?

lib/Transforms/Scalar/DeadStoreElimination.cpp

917 ↗

(On Diff #142230)

There are already existing functions for this, like isPowerOf2_32, can you please use one of them?

[dneilson]

lib/Transforms/Scalar/DeadStoreElimination.cpp

917 ↗

(On Diff #142230)

Perhaps a misleading comment. The line below this is checking whether NewLength is an integer multiple of ElementSize (i.e. whether NewLength % ElementSize == 0), but I did it via a bit operation because we know that ElementSize must be a power of 2 (via the spec for the atomic memory intrinsics).

I'll just use the modulo operator; less potential for confusion.

[dneilson]

• Use mod operator instead of bit twiddling

[dneilson]

ping - any takers to review this? I'm fairly certain it's correct, but a sanity check would be nice.

[efriedma]

LGTM

llvm/trunk/lib/Transforms/Scalar/DeadStoreElimination.cpp

static bool isShortenableAtTheEnd(Instruction *I) {
   if (isa<StoreInst>(I))
     return false;
 
   if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
     switch (II->getIntrinsicID()) {
       default: return false;
       case Intrinsic::memset:
       case Intrinsic::memcpy:
+      case Intrinsic::memcpy_element_unordered_atomic:
+      case Intrinsic::memset_element_unordered_atomic:
         // Do shorten memory intrinsics.
         // FIXME: Add memmove if it's also safe to transform.
-        // TODO: Add atomic memcpy/memset
         return true;
     }
   }
 
   // Don't shorten libcalls calls for now.
 
   return false;
 }
 
 /// Returns true if the beginning of this instruction can be safely shortened
 /// in length.
 static bool isShortenableAtTheBeginning(Instruction *I) {
   // FIXME: Handle only memset for now. Supporting memcpy/memmove should be
   // easily done by offsetting the source address.
-  // TODO: Handle atomic memory intrinsics
-  IntrinsicInst *II = dyn_cast<IntrinsicInst>(I);
-  return II && II->getIntrinsicID() == Intrinsic::memset;
+  return isa<AnyMemSetInst>(I);
 }
 
 /// Return the pointer that is being written to.
 static Value *getStoredPointerOperand(Instruction *I) {
   //TODO: factor this to reuse getLocForWrite
   MemoryLocation Loc = getLocForWrite(I);
   assert(Loc.Ptr &&
          "unable to find pointer writen for analyzable instruction?");
static bool tryToShorten(Instruction *EarlierWrite, int64_t &EarlierOffset,
                          int64_t &EarlierSize, int64_t LaterOffset,
                          int64_t LaterSize, bool IsOverwriteEnd) {
   // TODO: base this on the target vector size so that if the earlier
   // store was too small to get vector writes anyway then its likely
   // a good idea to shorten it
   // Power of 2 vector writes are probably always a bad idea to optimize
   // as any store/memset/memcpy is likely using vector instructions so
   // shortening it to not vector size is likely to be slower
-  MemIntrinsic *EarlierIntrinsic = cast<MemIntrinsic>(EarlierWrite);
+  auto *EarlierIntrinsic = cast<AnyMemIntrinsic>(EarlierWrite);
   unsigned EarlierWriteAlign = EarlierIntrinsic->getDestAlignment();
   if (!IsOverwriteEnd)
     LaterOffset = int64_t(LaterOffset + LaterSize);
 
   if (!(isPowerOf2_64(LaterOffset) && EarlierWriteAlign <= LaterOffset) &&
       !((EarlierWriteAlign != 0) && LaterOffset % EarlierWriteAlign == 0))
     return false;
 
+  int64_t NewLength = IsOverwriteEnd
+                          ? LaterOffset - EarlierOffset
+                          : EarlierSize - (LaterOffset - EarlierOffset);
+
+  if (auto *AMI = dyn_cast<AtomicMemIntrinsic>(EarlierWrite)) {
+    // When shortening an atomic memory intrinsic, the newly shortened
+    // length must remain an integer multiple of the element size.
+    const uint32_t ElementSize = AMI->getElementSizeInBytes();
+    if (0 != NewLength % ElementSize)
+      return false;
+  }
+
   DEBUG(dbgs() << "DSE: Remove Dead Store:\n  OW "
                << (IsOverwriteEnd ? "END" : "BEGIN") << ": " << *EarlierWrite
                << "\n  KILLER (offset " << LaterOffset << ", " << EarlierSize
                << ")\n");
 
-  int64_t NewLength = IsOverwriteEnd
-                          ? LaterOffset - EarlierOffset
-                          : EarlierSize - (LaterOffset - EarlierOffset);
-
   Value *EarlierWriteLength = EarlierIntrinsic->getLength();
   Value *TrimmedLength =
       ConstantInt::get(EarlierWriteLength->getType(), NewLength);
   EarlierIntrinsic->setLength(TrimmedLength);
 
   EarlierSize = NewLength;
   if (!IsOverwriteEnd) {
     int64_t OffsetMoved = (LaterOffset - EarlierOffset);

llvm/trunk/test/Transforms/DeadStoreElimination/OverwriteStoreBegin.ll

entry:
   ret void
 }
 
 define void @write4to7_atomic(i32* nocapture %p) {
 ; CHECK-LABEL: @write4to7_atomic(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
 ; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 28, i32 4)
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 4
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i32 4)
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 1
 ; CHECK-NEXT:    store atomic i32 1, i32* [[ARRAYIDX1]] unordered, align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %arrayidx0 = getelementptr inbounds i32, i32* %p, i64 1
   %p3 = bitcast i32* %arrayidx0 to i8*
   call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i32 4)
entry:
   store i32 1, i32* %p, align 4
   ret void
 }
 
 define void @write0to3_atomic(i32* nocapture %p) {
 ; CHECK-LABEL: @write0to3_atomic(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 28, i32 4)
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 4
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i32 4)
 ; CHECK-NEXT:    store atomic i32 1, i32* [[P]] unordered, align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %p3 = bitcast i32* %p to i8*
   call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i32 4)
   store atomic i32 1, i32* %p unordered, align 4
   ret void
 }
 
 ; Atomicity of the store is weaker from the memset
 define void @write0to3_atomic_weaker(i32* nocapture %p) {
 ; CHECK-LABEL: @write0to3_atomic_weaker(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 28, i32 4)
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 4
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i32 4)
 ; CHECK-NEXT:    store i32 1, i32* [[P]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %p3 = bitcast i32* %p to i8*
   call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i32 4)
   store i32 1, i32* %p, align 4
   ret void

 }
 
 ; Changing the memset start and length is okay here because the
 ; store is a multiple of the memset element size
 define void @write0to7_atomic(i32* nocapture %p) {
 ; CHECK-LABEL: @write0to7_atomic(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 32, i32 4)
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 8
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i32 4)
 ; CHECK-NEXT:    [[P4:%.*]] = bitcast i32* [[P]] to i64*
 ; CHECK-NEXT:    store atomic i64 1, i64* [[P4]] unordered, align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %p3 = bitcast i32* %p to i8*
   call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 32, i32 4)
   %p4 = bitcast i32* %p to i64*
entry:
   ret void
 }
 
 define void @write0to7_2_atomic(i32* nocapture %p) {
 ; CHECK-LABEL: @write0to7_2_atomic(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
 ; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 28, i32 4)
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 4
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i32 4)
 ; CHECK-NEXT:    [[P4:%.*]] = bitcast i32* [[P]] to i64*
 ; CHECK-NEXT:    store atomic i64 1, i64* [[P4]] unordered, align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %arrayidx0 = getelementptr inbounds i32, i32* %p, i64 1
   %p3 = bitcast i32* %arrayidx0 to i8*
   call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i32 4)
entry:
   ret void
 }
 
 define void @write8To15AndThen0To7_atomic(i64* nocapture %P) {
 ; CHECK-LABEL: @write8To15AndThen0To7_atomic(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[BASE0:%.*]] = bitcast i64* [[P:%.*]] to i8*
 ; CHECK-NEXT:    [[MYBASE0:%.*]] = getelementptr inbounds i8, i8* [[BASE0]], i64 0
-; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[MYBASE0]], i8 0, i64 32, i32 8)
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[MYBASE0]], i64 16
+; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[TMP0]], i8 0, i64 16, i32 8)
 ; CHECK-NEXT:    [[BASE64_0:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 0
 ; CHECK-NEXT:    [[BASE64_1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 1
 ; CHECK-NEXT:    store atomic i64 1, i64* [[BASE64_1]] unordered, align 8
 ; CHECK-NEXT:    store atomic i64 2, i64* [[BASE64_0]] unordered, align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
 
entry:
   ret void
 }
 
 define void @write8To15AndThen0To7_atomic_weaker(i64* nocapture %P) {
 ; CHECK-LABEL: @write8To15AndThen0To7_atomic_weaker(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[BASE0:%.*]] = bitcast i64* [[P:%.*]] to i8*
 ; CHECK-NEXT:    [[MYBASE0:%.*]] = getelementptr inbounds i8, i8* [[BASE0]], i64 0
-; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[MYBASE0]], i8 0, i64 32, i32 8)
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[MYBASE0]], i64 16
+; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[TMP0]], i8 0, i64 16, i32 8)
 ; CHECK-NEXT:    [[BASE64_0:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 0
 ; CHECK-NEXT:    [[BASE64_1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 1
 ; CHECK-NEXT:    store atomic i64 1, i64* [[BASE64_1]] unordered, align 8
 ; CHECK-NEXT:    store i64 2, i64* [[BASE64_0]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
 
entry:
   ret void
 }
 
 define void @write8To15AndThen0To7_atomic_weaker_2(i64* nocapture %P) {
 ; CHECK-LABEL: @write8To15AndThen0To7_atomic_weaker_2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[BASE0:%.*]] = bitcast i64* [[P:%.*]] to i8*
 ; CHECK-NEXT:    [[MYBASE0:%.*]] = getelementptr inbounds i8, i8* [[BASE0]], i64 0
-; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[MYBASE0]], i8 0, i64 32, i32 8)
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[MYBASE0]], i64 16
+; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[TMP0]], i8 0, i64 16, i32 8)
 ; CHECK-NEXT:    [[BASE64_0:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 0
 ; CHECK-NEXT:    [[BASE64_1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 1
 ; CHECK-NEXT:    store i64 1, i64* [[BASE64_1]], align 8
 ; CHECK-NEXT:    store atomic i64 2, i64* [[BASE64_0]] unordered, align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
 

llvm/trunk/test/Transforms/DeadStoreElimination/OverwriteStoreEnd.ll

entry:
   ret void
 }
 
 define void @write24to28_atomic(i32* nocapture %p) nounwind uwtable ssp {
 ; CHECK-LABEL: @write24to28_atomic(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
 ; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 28, i32 4)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 24, i32 4)
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
 ; CHECK-NEXT:    store atomic i32 1, i32* [[ARRAYIDX1]] unordered, align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %arrayidx0 = getelementptr inbounds i32, i32* %p, i64 1
   %p3 = bitcast i32* %arrayidx0 to i8*
   call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i32 4)
   %arrayidx1 = getelementptr inbounds i32, i32* %p, i64 7
   store atomic i32 1, i32* %arrayidx1 unordered, align 4
   ret void
 }
 
 ; Atomicity of the store is weaker from the memset
 define void @write24to28_atomic_weaker(i32* nocapture %p) nounwind uwtable ssp {
 ; CHECK-LABEL: @write24to28_atomic_weaker(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
 ; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 28, i32 4)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 24, i32 4)
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
 ; CHECK-NEXT:    store i32 1, i32* [[ARRAYIDX1]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %arrayidx0 = getelementptr inbounds i32, i32* %p, i64 1
   %p3 = bitcast i32* %arrayidx0 to i8*
   call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i32 4)
entry:
   store i32 1, i32* %arrayidx1, align 4
   ret void
 }
 
 define void @write28to32_atomic(i32* nocapture %p) nounwind uwtable ssp {
 ; CHECK-LABEL: @write28to32_atomic(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 32, i32 4)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 28, i32 4)
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
 ; CHECK-NEXT:    store atomic i32 1, i32* [[ARRAYIDX1]] unordered, align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %p3 = bitcast i32* %p to i8*
   call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 32, i32 4)
   %arrayidx1 = getelementptr inbounds i32, i32* %p, i64 7
entry:
   store i32 1, i32* %c, align 4
   ret void
 }
 
 define void @write32to36_atomic(%struct.vec2plusi* nocapture %p) nounwind uwtable ssp {
 ; CHECK-LABEL: @write32to36_atomic(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %struct.vec2plusi* [[P:%.*]] to i8*
-; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 [[TMP0]], i8* align 16 bitcast (%struct.vec2plusi* @glob2 to i8*), i64 36, i32 4)
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 [[TMP0]], i8* align 16 bitcast (%struct.vec2plusi* @glob2 to i8*), i64 32, i32 4)
 ; CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [[STRUCT_VEC2PLUSI:%.*]], %struct.vec2plusi* [[P]], i64 0, i32 2
 ; CHECK-NEXT:    store atomic i32 1, i32* [[C]] unordered, align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %0 = bitcast %struct.vec2plusi* %p to i8*
   tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 %0, i8* align 16 bitcast (%struct.vec2plusi* @glob2 to i8*), i64 36, i32 4)
   %c = getelementptr inbounds %struct.vec2plusi, %struct.vec2plusi* %p, i64 0, i32 2
   store atomic i32 1, i32* %c unordered, align 4
   ret void
 }
 
 ; Atomicity of the store is weaker than the memcpy
 define void @write32to36_atomic_weaker(%struct.vec2plusi* nocapture %p) nounwind uwtable ssp {
 ; CHECK-LABEL: @write32to36_atomic_weaker(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %struct.vec2plusi* [[P:%.*]] to i8*
-; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 [[TMP0]], i8* align 16 bitcast (%struct.vec2plusi* @glob2 to i8*), i64 36, i32 4)
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 [[TMP0]], i8* align 16 bitcast (%struct.vec2plusi* @glob2 to i8*), i64 32, i32 4)
 ; CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [[STRUCT_VEC2PLUSI:%.*]], %struct.vec2plusi* [[P]], i64 0, i32 2
 ; CHECK-NEXT:    store i32 1, i32* [[C]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %0 = bitcast %struct.vec2plusi* %p to i8*
   tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 %0, i8* align 16 bitcast (%struct.vec2plusi* @glob2 to i8*), i64 36, i32 4)
   %c = getelementptr inbounds %struct.vec2plusi, %struct.vec2plusi* %p, i64 0, i32 2
entry:
   store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32>* %c, align 4
   ret void
 }
 
 define void @write16to32_atomic(%struct.vec2* nocapture %p) nounwind uwtable ssp {
 ; CHECK-LABEL: @write16to32_atomic(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %struct.vec2* [[P:%.*]] to i8*
-; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 [[TMP0]], i8* align 16 bitcast (%struct.vec2* @glob1 to i8*), i64 32, i32 4)
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 [[TMP0]], i8* align 16 bitcast (%struct.vec2* @glob1 to i8*), i64 16, i32 4)
 ; CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [[STRUCT_VEC2:%.*]], %struct.vec2* [[P]], i64 0, i32 1
 ; CHECK-NEXT:    store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32>* [[C]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %0 = bitcast %struct.vec2* %p to i8*
   tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 %0, i8* align 16 bitcast (%struct.vec2* @glob1 to i8*), i64 32, i32 4)
   %c = getelementptr inbounds %struct.vec2, %struct.vec2* %p, i64 0, i32 1
entry:
   ret void
 }
 
 define void @write16To23AndThen24To31_atomic(i64* nocapture %P, i64 %n64, i32 %n32, i16 %n16, i8 %n8) {
 ; CHECK-LABEL: @write16To23AndThen24To31_atomic(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[BASE0:%.*]] = bitcast i64* [[P:%.*]] to i8*
 ; CHECK-NEXT:    [[MYBASE0:%.*]] = getelementptr inbounds i8, i8* [[BASE0]], i64 0
-; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[MYBASE0]], i8 0, i64 32, i32 8)
+; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[MYBASE0]], i8 0, i64 16, i32 8)
 ; CHECK-NEXT:    [[BASE64_2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 2
 ; CHECK-NEXT:    [[BASE64_3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 3
 ; CHECK-NEXT:    store atomic i64 3, i64* [[BASE64_2]] unordered, align 8
 ; CHECK-NEXT:    store atomic i64 3, i64* [[BASE64_3]] unordered, align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
 
entry:
   ret void
 }
 
 define void @write16To23AndThen24To31_atomic_weaker1(i64* nocapture %P, i64 %n64, i32 %n32, i16 %n16, i8 %n8) {
 ; CHECK-LABEL: @write16To23AndThen24To31_atomic_weaker1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[BASE0:%.*]] = bitcast i64* [[P:%.*]] to i8*
 ; CHECK-NEXT:    [[MYBASE0:%.*]] = getelementptr inbounds i8, i8* [[BASE0]], i64 0
-; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[MYBASE0]], i8 0, i64 32, i32 8)
+; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[MYBASE0]], i8 0, i64 16, i32 8)
 ; CHECK-NEXT:    [[BASE64_2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 2
 ; CHECK-NEXT:    [[BASE64_3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 3
 ; CHECK-NEXT:    store i64 3, i64* [[BASE64_2]], align 8
 ; CHECK-NEXT:    store atomic i64 3, i64* [[BASE64_3]] unordered, align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
 
entry:
   ret void
 }
 
 define void @write16To23AndThen24To31_atomic_weaker2(i64* nocapture %P, i64 %n64, i32 %n32, i16 %n16, i8 %n8) {
 ; CHECK-LABEL: @write16To23AndThen24To31_atomic_weaker2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[BASE0:%.*]] = bitcast i64* [[P:%.*]] to i8*
 ; CHECK-NEXT:    [[MYBASE0:%.*]] = getelementptr inbounds i8, i8* [[BASE0]], i64 0
-; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[MYBASE0]], i8 0, i64 32, i32 8)
+; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[MYBASE0]], i8 0, i64 16, i32 8)
 ; CHECK-NEXT:    [[BASE64_2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 2
 ; CHECK-NEXT:    [[BASE64_3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 3
 ; CHECK-NEXT:    store atomic i64 3, i64* [[BASE64_2]] unordered, align 8
 ; CHECK-NEXT:    store i64 3, i64* [[BASE64_3]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry: