diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp @@ -801,12 +801,14 @@ // // FIXME: We will upgrade the alignment of the alloca even if it turns out // we can't vectorize for some other reason. + Value *PtrOperand = getLoadStorePointerOperand(C[CBegin].Inst); + bool IsAllocaAccess = isa(PtrOperand->stripInBoundsOffsets()); Align Alignment = getLoadStoreAlignment(C[CBegin].Inst); - if (AS == DL.getAllocaAddrSpace() && Alignment.value() % SizeBytes != 0 && - IsAllowedAndFast(Align(StackAdjustedAlignment))) { + Align PrefAlign = Align(StackAdjustedAlignment); + if (IsAllocaAccess && AS == DL.getAllocaAddrSpace() && + Alignment.value() % SizeBytes != 0 && IsAllowedAndFast(PrefAlign)) { Align NewAlign = getOrEnforceKnownAlignment( - getLoadStorePointerOperand(C[CBegin].Inst), - Align(StackAdjustedAlignment), DL, C[CBegin].Inst, nullptr, &DT); + PtrOperand, PrefAlign, DL, C[CBegin].Inst, nullptr, &DT); if (NewAlign >= Alignment) { LLVM_DEBUG(dbgs() << "LSV: splitByChain upgrading alloca alignment from " diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll --- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll +++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll @@ -21,9 +21,9 @@ ; UNALIGNED-LABEL: @load_unknown_offset_align1_i8( ; UNALIGNED-NEXT: [[ALLOCA:%.*]] = alloca [128 x i8], align 1, addrspace(5) ; UNALIGNED-NEXT: [[PTR0:%.*]] = getelementptr inbounds [128 x i8], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[OFFSET:%.*]] -; UNALIGNED-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr addrspace(5) [[PTR0]], align 1 -; UNALIGNED-NEXT: [[VAL01:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0 -; UNALIGNED-NEXT: [[VAL12:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1 +; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x i8>, ptr addrspace(5) [[PTR0]], align 1 +; UNALIGNED-NEXT: [[VAL01:%.*]] = extractelement <2 x i8> [[TMP1]], i32 0 +; UNALIGNED-NEXT: [[VAL12:%.*]] = extractelement <2 x i8> [[TMP1]], i32 1 ; UNALIGNED-NEXT: [[ADD:%.*]] = add i8 [[VAL01]], [[VAL12]] ; UNALIGNED-NEXT: store i8 [[ADD]], ptr addrspace(1) [[OUT:%.*]], align 1 ; UNALIGNED-NEXT: ret void @@ -52,9 +52,9 @@ ; UNALIGNED-LABEL: @load_unknown_offset_align1_i16( ; UNALIGNED-NEXT: [[ALLOCA:%.*]] = alloca [128 x i16], align 1, addrspace(5) ; UNALIGNED-NEXT: [[PTR0:%.*]] = getelementptr inbounds [128 x i16], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[OFFSET:%.*]] -; UNALIGNED-NEXT: [[TMP2:%.*]] = load <2 x i16>, ptr addrspace(5) [[PTR0]], align 1 -; UNALIGNED-NEXT: [[VAL01:%.*]] = extractelement <2 x i16> [[TMP2]], i32 0 -; UNALIGNED-NEXT: [[VAL12:%.*]] = extractelement <2 x i16> [[TMP2]], i32 1 +; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x i16>, ptr addrspace(5) [[PTR0]], align 1 +; UNALIGNED-NEXT: [[VAL01:%.*]] = extractelement <2 x i16> [[TMP1]], i32 0 +; UNALIGNED-NEXT: [[VAL12:%.*]] = extractelement <2 x i16> [[TMP1]], i32 1 ; UNALIGNED-NEXT: [[ADD:%.*]] = add i16 [[VAL01]], [[VAL12]] ; UNALIGNED-NEXT: store i16 [[ADD]], ptr addrspace(1) [[OUT:%.*]], align 2 ; UNALIGNED-NEXT: ret void @@ -85,9 +85,9 @@ ; UNALIGNED-LABEL: @load_unknown_offset_align1_i32( ; UNALIGNED-NEXT: [[ALLOCA:%.*]] = alloca [128 x i32], align 1, addrspace(5) ; UNALIGNED-NEXT: [[PTR0:%.*]] = getelementptr inbounds [128 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[OFFSET:%.*]] -; UNALIGNED-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr addrspace(5) [[PTR0]], align 1 -; UNALIGNED-NEXT: [[VAL01:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0 -; UNALIGNED-NEXT: [[VAL12:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1 +; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(5) [[PTR0]], align 1 +; UNALIGNED-NEXT: [[VAL01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 +; UNALIGNED-NEXT: [[VAL12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 ; UNALIGNED-NEXT: [[ADD:%.*]] = add i32 [[VAL01]], [[VAL12]] ; UNALIGNED-NEXT: store i32 [[ADD]], ptr addrspace(1) [[OUT:%.*]], align 4 ; UNALIGNED-NEXT: ret void @@ -107,9 +107,9 @@ ; CHECK-LABEL: @load_alloca16_unknown_offset_align1_i32( ; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [128 x i32], align 16, addrspace(5) ; CHECK-NEXT: [[PTR0:%.*]] = getelementptr inbounds [128 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[OFFSET:%.*]] -; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr addrspace(5) [[PTR0]], align 4 -; CHECK-NEXT: [[VAL01:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0 -; CHECK-NEXT: [[VAL12:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1 +; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(5) [[PTR0]], align 4 +; CHECK-NEXT: [[VAL01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 +; CHECK-NEXT: [[VAL12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 ; CHECK-NEXT: [[ADD:%.*]] = add i32 [[VAL01]], [[VAL12]] ; CHECK-NEXT: store i32 [[ADD]], ptr addrspace(1) [[OUT:%.*]], align 4 ; CHECK-NEXT: ret void @@ -235,11 +235,11 @@ define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i32() { ; CHECK-LABEL: @merge_private_load_4_vector_elts_loads_v4i32( ; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [8 x i32], align 4, addrspace(5) -; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(5) [[ALLOCA]], align 4 -; CHECK-NEXT: [[LOAD01:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0 -; CHECK-NEXT: [[LOAD12:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1 -; CHECK-NEXT: [[LOAD23:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2 -; CHECK-NEXT: [[LOAD34:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3 +; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr addrspace(5) [[ALLOCA]], align 4 +; CHECK-NEXT: [[LOAD01:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0 +; CHECK-NEXT: [[LOAD12:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1 +; CHECK-NEXT: [[LOAD23:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2 +; CHECK-NEXT: [[LOAD34:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3 ; CHECK-NEXT: ret void ; %alloca = alloca [8 x i32], align 1, addrspace(5) @@ -257,11 +257,11 @@ define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i8() { ; CHECK-LABEL: @merge_private_load_4_vector_elts_loads_v4i8( ; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [8 x i8], align 4, addrspace(5) -; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr addrspace(5) [[ALLOCA]], align 4 -; CHECK-NEXT: [[LOAD01:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0 -; CHECK-NEXT: [[LOAD12:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1 -; CHECK-NEXT: [[LOAD23:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2 -; CHECK-NEXT: [[LOAD34:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3 +; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(5) [[ALLOCA]], align 4 +; CHECK-NEXT: [[LOAD01:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0 +; CHECK-NEXT: [[LOAD12:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1 +; CHECK-NEXT: [[LOAD23:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2 +; CHECK-NEXT: [[LOAD34:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3 ; CHECK-NEXT: ret void ; %alloca = alloca [8 x i8], align 1, addrspace(5) @@ -278,12 +278,15 @@ ; Make sure we don't think the alignment will increase if the base address isn't an alloca define void @private_store_2xi16_align2_not_alloca(ptr addrspace(5) %p, ptr addrspace(5) %r) #0 { -; CHECK-LABEL: @private_store_2xi16_align2_not_alloca( -; ALIGNED-NEXT: [[GEP_R:%.*]] = getelementptr i16, ptr addrspace(5) [[R:%.*]], i32 1 -; ALIGNED-NEXT: store i16 1, ptr addrspace(5) [[R]], align 2 -; ALIGNED-NEXT: store i16 2, ptr addrspace(5) [[GEP_R]], align 2 -; UNALIGNED-NEXT:store <2 x i16> -; CHECK-NEXT: ret void +; ALIGNED-LABEL: @private_store_2xi16_align2_not_alloca( +; ALIGNED-NEXT: [[GEP_R:%.*]] = getelementptr i16, ptr addrspace(5) [[R:%.*]], i32 1 +; ALIGNED-NEXT: store i16 1, ptr addrspace(5) [[R]], align 2 +; ALIGNED-NEXT: store i16 2, ptr addrspace(5) [[GEP_R]], align 2 +; ALIGNED-NEXT: ret void +; +; UNALIGNED-LABEL: @private_store_2xi16_align2_not_alloca( +; UNALIGNED-NEXT: store <2 x i16> , ptr addrspace(5) [[R:%.*]], align 2 +; UNALIGNED-NEXT: ret void ; %gep.r = getelementptr i16, ptr addrspace(5) %r, i32 1 store i16 1, ptr addrspace(5) %r, align 2 @@ -309,16 +312,25 @@ } define i32 @private_load_2xi16_align2_not_alloca(ptr addrspace(5) %p) #0 { -; CHECK-LABEL: @private_load_2xi16_align2_not_alloca( -; ALIGNED-NEXT: [[GEP_P:%.*]] = getelementptr i16, ptr addrspace(5) [[P:%.*]], i64 1 -; ALIGNED-NEXT: [[P_0:%.*]] = load i16, ptr addrspace(5) [[P]], align 2 -; ALIGNED-NEXT: [[P_1:%.*]] = load i16, ptr addrspace(5) [[GEP_P]], align 2 -; UNALIGNED-NEXT:load <2 x i16> -; CHECK: [[ZEXT_0:%.*]] = zext i16 -; CHECK-NEXT: [[ZEXT_1:%.*]] = zext i16 -; CHECK-NEXT: [[SHL_1:%.*]] = shl i32 [[ZEXT_1]], 16 -; CHECK-NEXT: [[OR:%.*]] = or i32 [[ZEXT_0]], [[SHL_1]] -; CHECK-NEXT: ret i32 [[OR]] +; ALIGNED-LABEL: @private_load_2xi16_align2_not_alloca( +; ALIGNED-NEXT: [[GEP_P:%.*]] = getelementptr i16, ptr addrspace(5) [[P:%.*]], i64 1 +; ALIGNED-NEXT: [[P_0:%.*]] = load i16, ptr addrspace(5) [[P]], align 2 +; ALIGNED-NEXT: [[P_1:%.*]] = load i16, ptr addrspace(5) [[GEP_P]], align 2 +; ALIGNED-NEXT: [[ZEXT_0:%.*]] = zext i16 [[P_0]] to i32 +; ALIGNED-NEXT: [[ZEXT_1:%.*]] = zext i16 [[P_1]] to i32 +; ALIGNED-NEXT: [[SHL_1:%.*]] = shl i32 [[ZEXT_1]], 16 +; ALIGNED-NEXT: [[OR:%.*]] = or i32 [[ZEXT_0]], [[SHL_1]] +; ALIGNED-NEXT: ret i32 [[OR]] +; +; UNALIGNED-LABEL: @private_load_2xi16_align2_not_alloca( +; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x i16>, ptr addrspace(5) [[P:%.*]], align 2 +; UNALIGNED-NEXT: [[P_01:%.*]] = extractelement <2 x i16> [[TMP1]], i32 0 +; UNALIGNED-NEXT: [[P_12:%.*]] = extractelement <2 x i16> [[TMP1]], i32 1 +; UNALIGNED-NEXT: [[ZEXT_0:%.*]] = zext i16 [[P_01]] to i32 +; UNALIGNED-NEXT: [[ZEXT_1:%.*]] = zext i16 [[P_12]] to i32 +; UNALIGNED-NEXT: [[SHL_1:%.*]] = shl i32 [[ZEXT_1]], 16 +; UNALIGNED-NEXT: [[OR:%.*]] = or i32 [[ZEXT_0]], [[SHL_1]] +; UNALIGNED-NEXT: ret i32 [[OR]] ; %gep.p = getelementptr i16, ptr addrspace(5) %p, i64 1 %p.0 = load i16, ptr addrspace(5) %p, align 2 @@ -342,9 +354,9 @@ ; ALIGNED-NEXT: ret i32 [[OR]] ; ; UNALIGNED-LABEL: @private_load_2xi16_align1_not_alloca( -; UNALIGNED-NEXT: [[TMP2:%.*]] = load <2 x i16>, ptr addrspace(5) [[P:%.*]], align 1 -; UNALIGNED-NEXT: [[P_01:%.*]] = extractelement <2 x i16> [[TMP2]], i32 0 -; UNALIGNED-NEXT: [[P_12:%.*]] = extractelement <2 x i16> [[TMP2]], i32 1 +; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x i16>, ptr addrspace(5) [[P:%.*]], align 1 +; UNALIGNED-NEXT: [[P_01:%.*]] = extractelement <2 x i16> [[TMP1]], i32 0 +; UNALIGNED-NEXT: [[P_12:%.*]] = extractelement <2 x i16> [[TMP1]], i32 1 ; UNALIGNED-NEXT: [[ZEXT_0:%.*]] = zext i16 [[P_01]] to i32 ; UNALIGNED-NEXT: [[ZEXT_1:%.*]] = zext i16 [[P_12]] to i32 ; UNALIGNED-NEXT: [[SHL_1:%.*]] = shl i32 [[ZEXT_1]], 16