[LSV] Change the default value of InstertElement to poison

[hyeongyukim]

Summary

This patch is changing the InsertElement's placeholder to poison without changing the LSV's behavior.

Regardless of whether StoreTy is FixedVectorType or not, the poison value will be overwritten with a different value.
Therefore, whether the InsertElement's placeholder is poison or undef will not affect the result of the program.

[hyeongyukim]

llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp

1131

if StoreTy is FixedVectorType, the poison will be hided by this CreateInsertElement.
Since VecTy's length is Chain.size() * VecStoreTy->getNumElements(), all poisons will be hidden.

1144

if StoreTy is not FixedVectorType, the poison will be hided by this CreateInsertElement.
Since VecTy's length is Chain.size(), all poisons will be hidden.

[nikic]

LGTM

[hyeongyukim]

Thanks!

llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp

if (!TTI.isLegalToVectorizeStoreChain(SzInBytes, Alignment, AS)) {
     return vectorizeStoreChain(Chains.first, InstructionsProcessed) |
            vectorizeStoreChain(Chains.second, InstructionsProcessed);
   }
 
   BasicBlock::iterator First, Last;
   std::tie(First, Last) = getBoundaryInstrs(Chain);
   Builder.SetInsertPoint(&*Last);
 
-  Value *Vec = UndefValue::get(VecTy);
+  Value *Vec = PoisonValue::get(VecTy);
 
   if (VecStoreTy) {
     unsigned VecWidth = VecStoreTy->getNumElements();
     for (unsigned I = 0, E = Chain.size(); I != E; ++I) {
       StoreInst *Store = cast<StoreInst>(Chain[I]);
       for (unsigned J = 0, NE = VecStoreTy->getNumElements(); J != NE; ++J) {
         unsigned NewIdx = J + I * VecWidth;
         Value *Extract = Builder.CreateExtractElement(Store->getValueOperand(),
                                                       Builder.getInt32(J));
         if (Extract->getType() != StoreTy->getScalarType())
           Extract = Builder.CreateBitCast(Extract, StoreTy->getScalarType());
 
         Value *Insert =
             Builder.CreateInsertElement(Vec, Extract, Builder.getInt32(NewIdx));
         Vec = Insert;
       }
     }
   } else {
     for (unsigned I = 0, E = Chain.size(); I != E; ++I) {
       StoreInst *Store = cast<StoreInst>(Chain[I]);
       Value *Extract = Store->getValueOperand();
       if (Extract->getType() != StoreTy->getScalarType())
         Extract =
             Builder.CreateBitOrPointerCast(Extract, StoreTy->getScalarType());
 
       Value *Insert =
           Builder.CreateInsertElement(Vec, Extract, Builder.getInt32(I));
       Vec = Insert;
     }
   }
 
   StoreInst *SI = Builder.CreateAlignedStore(
     Vec,
     Builder.CreateBitCast(S0->getPointerOperand(), VecTy->getPointerTo(AS)),
     Alignment);

llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll

define amdgpu_kernel void @merge_global_store_4_constants_i64(i64 addrspace(1)* %out) #0 {
   store i64 1234, i64 addrspace(1)* %out
   ret void
 }
 
 ; CHECK-LABEL: @merge_global_store_2_adjacent_loads_i32
 ; CHECK: [[LOAD:%[^ ]+]] = load <2 x i32>
 ; CHECK: [[ELT0:%[^ ]+]] = extractelement <2 x i32> [[LOAD]], i32 0
 ; CHECK: [[ELT1:%[^ ]+]] = extractelement <2 x i32> [[LOAD]], i32 1
-; CHECK: [[INSERT0:%[^ ]+]] = insertelement <2 x i32> undef, i32 [[ELT0]], i32 0
+; CHECK: [[INSERT0:%[^ ]+]] = insertelement <2 x i32> poison, i32 [[ELT0]], i32 0
 ; CHECK: [[INSERT1:%[^ ]+]] = insertelement <2 x i32> [[INSERT0]], i32 [[ELT1]], i32 1
 ; CHECK: store <2 x i32> [[INSERT1]]
 define amdgpu_kernel void @merge_global_store_2_adjacent_loads_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
   %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
 
   %lo = load i32, i32 addrspace(1)* %in
   %hi = load i32, i32 addrspace(1)* %in.gep.1
define amdgpu_kernel void @merge_global_store_2_adjacent_loads_i32_nonzero_base(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   store i32 %hi, i32 addrspace(1)* %out.gep.1
   ret void
 }
 
 ; CHECK-LABEL: @merge_global_store_2_adjacent_loads_shuffle_i32
 ; CHECK: [[LOAD:%[^ ]+]] = load <2 x i32>
 ; CHECK: [[ELT0:%[^ ]+]] = extractelement <2 x i32> [[LOAD]], i32 0
 ; CHECK: [[ELT1:%[^ ]+]] = extractelement <2 x i32> [[LOAD]], i32 1
-; CHECK: [[INSERT0:%[^ ]+]] = insertelement <2 x i32> undef, i32 [[ELT1]], i32 0
+; CHECK: [[INSERT0:%[^ ]+]] = insertelement <2 x i32> poison, i32 [[ELT1]], i32 0
 ; CHECK: [[INSERT1:%[^ ]+]] = insertelement <2 x i32> [[INSERT0]], i32 [[ELT0]], i32 1
 ; CHECK: store <2 x i32> [[INSERT1]]
 define amdgpu_kernel void @merge_global_store_2_adjacent_loads_shuffle_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
   %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
 
   %lo = load i32, i32 addrspace(1)* %in
   %hi = load i32, i32 addrspace(1)* %in.gep.1

llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll

entry:
   %ld.0 = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %a.cast
   %ld.1 = load i64, i64 addrspace(1)* %a.1
 
   ret void
 }
 
 ; CHECK-LABEL: @merge_store_ptr64_i64(
 ; CHECK: [[ELT0:%[^ ]+]] = ptrtoint i8 addrspace(1)* %ptr0 to i64
-; CHECK: insertelement <2 x i64> undef, i64 [[ELT0]], i32 0
+; CHECK: insertelement <2 x i64> poison, i64 [[ELT0]], i32 0
 ; CHECK: store <2 x i64>
 define amdgpu_kernel void @merge_store_ptr64_i64(i64 addrspace(1)* nocapture %a, i8 addrspace(1)* %ptr0, i64 %val1) #0 {
 entry:
   %a.cast = bitcast i64 addrspace(1)* %a to i8 addrspace(1)* addrspace(1)*
   %a.1 = getelementptr inbounds i64, i64 addrspace(1)* %a, i64 1
 
 
   store i8 addrspace(1)* %ptr0, i8 addrspace(1)* addrspace(1)* %a.cast
entry:
   %ld.0 = load i8 addrspace(3)*, i8 addrspace(3)* addrspace(3)* %a.cast
   %ld.1 = load i32, i32 addrspace(3)* %a.1
 
   ret void
 }
 
 ; CHECK-LABEL: @merge_store_ptr32_i32(
 ; CHECK: [[ELT0:%[^ ]+]] = ptrtoint i8 addrspace(3)* %ptr0 to i32
-; CHECK: insertelement <2 x i32> undef, i32 [[ELT0]], i32 0
+; CHECK: insertelement <2 x i32> poison, i32 [[ELT0]], i32 0
 ; CHECK: store <2 x i32>
 define amdgpu_kernel void @merge_store_ptr32_i32(i32 addrspace(3)* nocapture %a, i8 addrspace(3)* %ptr0, i32 %val1) #0 {
 entry:
   %a.cast = bitcast i32 addrspace(3)* %a to i8 addrspace(3)* addrspace(3)*
   %a.1 = getelementptr inbounds i32, i32 addrspace(3)* %a, i32 1
 
   store i8 addrspace(3)* %ptr0, i8 addrspace(3)* addrspace(3)* %a.cast
   store i32 %val1, i32 addrspace(3)* %a.1
entry:
   %ld.0 = load double, double addrspace(1)* %a
   %ld.1 = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %a.1.cast
 
   ret void
 }
 
 ; CHECK-LABEL: @merge_store_ptr64_f64(
 ; CHECK: [[ELT0_INT:%[^ ]+]] = ptrtoint i8 addrspace(1)* %ptr0 to i64
-; CHECK: insertelement <2 x i64> undef, i64 [[ELT0_INT]], i32 0
+; CHECK: insertelement <2 x i64> poison, i64 [[ELT0_INT]], i32 0
 ; CHECK: [[ELT1_INT:%[^ ]+]] = bitcast double %val1 to i64
 ; CHECK: insertelement <2 x i64> %{{[^ ]+}}, i64 [[ELT1_INT]], i32 1
 ; CHECK: store <2 x i64>
 define amdgpu_kernel void @merge_store_ptr64_f64(double addrspace(1)* nocapture %a, i8 addrspace(1)* %ptr0, double %val1) #0 {
 entry:
   %a.cast = bitcast double addrspace(1)* %a to i8 addrspace(1)* addrspace(1)*
   %a.1 = getelementptr inbounds double, double addrspace(1)* %a, i64 1
 
   store i8 addrspace(1)* %ptr0, i8 addrspace(1)* addrspace(1)* %a.cast
   store double %val1, double addrspace(1)* %a.1
 
   ret void
 }
 
 ; CHECK-LABEL: @merge_store_f64_ptr64(
 ; CHECK: [[ELT0_INT:%[^ ]+]] = bitcast double %val0 to i64
-; CHECK: insertelement <2 x i64> undef, i64 [[ELT0_INT]], i32 0
+; CHECK: insertelement <2 x i64> poison, i64 [[ELT0_INT]], i32 0
 ; CHECK: [[ELT1_INT:%[^ ]+]] = ptrtoint i8 addrspace(1)* %ptr1 to i64
 ; CHECK: insertelement <2 x i64> %{{[^ ]+}}, i64 [[ELT1_INT]], i32 1
 ; CHECK: store <2 x i64>
 define amdgpu_kernel void @merge_store_f64_ptr64(i8 addrspace(1)* addrspace(1)* nocapture %a, double %val0, i8 addrspace(1)* %ptr1) #0 {
 entry:
   %a.1 = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %a, i64 1
   %a.cast = bitcast i8 addrspace(1)* addrspace(1)* %a to double addrspace(1)*
 
   store double %val0, double addrspace(1)* %a.cast
   store i8 addrspace(1)* %ptr1, i8 addrspace(1)* addrspace(1)* %a.1
 
   ret void
 }
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }

llvm/test/Transforms/LoadStoreVectorizer/int_sideeffect.ll

 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[P0]] to <4 x float>*
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 16
 ; CHECK-NEXT:    [[L01:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
 ; CHECK-NEXT:    [[L12:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
 ; CHECK-NEXT:    [[L23:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
 ; CHECK-NEXT:    [[L34:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
 ; CHECK-NEXT:    call void @llvm.sideeffect()
 ; CHECK-NEXT:    call void @llvm.sideeffect()
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> undef, float [[L01]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> poison, float [[L01]], i32 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[L12]], i32 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x float> [[TMP4]], float [[L23]], i32 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[L34]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[P0]] to <4 x float>*
 ; CHECK-NEXT:    store <4 x float> [[TMP6]], <4 x float>* [[TMP7]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %p0 = getelementptr float, float* %p, i64 0

 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[P0]] to <4 x float>*
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 16
 ; CHECK-NEXT:    [[L01:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
 ; CHECK-NEXT:    [[L12:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
 ; CHECK-NEXT:    [[L23:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
 ; CHECK-NEXT:    [[L34:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
 ; CHECK-NEXT:    call void @foo() #[[ATTR1:[0-9]+]]
 ; CHECK-NEXT:    call void @foo() #[[ATTR1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> undef, float [[L01]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> poison, float [[L01]], i32 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[L12]], i32 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x float> [[TMP4]], float [[L23]], i32 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[L34]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[P0]] to <4 x float>*
 ; CHECK-NEXT:    store <4 x float> [[TMP6]], <4 x float>* [[TMP7]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %p0 = getelementptr float, float* %p, i64 0