Index: include/llvm/LinkAllPasses.h =================================================================== --- include/llvm/LinkAllPasses.h +++ include/llvm/LinkAllPasses.h @@ -181,7 +181,7 @@ (void) llvm::createInstructionSimplifierPass(); (void) llvm::createLoopVectorizePass(); (void) llvm::createSLPVectorizerPass(); - (void) llvm::createLoadStoreVectorizerPass(128); + (void) llvm::createLoadStoreVectorizerPass(); (void) llvm::createBBVectorizePass(); (void) llvm::createPartiallyInlineLibCallsPass(); (void) llvm::createScalarizerPass(); Index: include/llvm/Transforms/Vectorize.h =================================================================== --- include/llvm/Transforms/Vectorize.h +++ include/llvm/Transforms/Vectorize.h @@ -145,7 +145,7 @@ // LoadStoreVectorizer - Create vector loads and stores, but leave scalar // operations. // -Pass *createLoadStoreVectorizerPass(unsigned VecRegSize); +Pass *createLoadStoreVectorizerPass(); } // End llvm namespace Index: lib/Transforms/Vectorize/LoadStoreVectorizer.cpp =================================================================== --- lib/Transforms/Vectorize/LoadStoreVectorizer.cpp +++ lib/Transforms/Vectorize/LoadStoreVectorizer.cpp @@ -47,17 +47,18 @@ AliasAnalysis &AA; DominatorTree &DT; ScalarEvolution &SE; + TargetTransformInfo &TTI; const DataLayout &DL; IRBuilder<> Builder; ValueListMap StoreRefs; ValueListMap LoadRefs; - unsigned VecRegSize; public: Vectorizer(Function &F, AliasAnalysis &AA, DominatorTree &DT, - ScalarEvolution &SE, unsigned VecRegSize) - : F(F), AA(AA), DT(DT), SE(SE), DL(F.getParent()->getDataLayout()), - Builder(SE.getContext()), VecRegSize(VecRegSize) {} + ScalarEvolution &SE, TargetTransformInfo &TTI) + : F(F), AA(AA), DT(DT), SE(SE), TTI(TTI), + DL(F.getParent()->getDataLayout()), + Builder(SE.getContext()) {} bool run(); @@ -109,10 +110,8 @@ class LoadStoreVectorizer : public FunctionPass { public: static char ID; - unsigned VecRegSize; - LoadStoreVectorizer(unsigned VecRegSize = 128) : FunctionPass(ID), - VecRegSize(VecRegSize) { + LoadStoreVectorizer() : FunctionPass(ID) { initializeLoadStoreVectorizerPass(*PassRegistry::getPassRegistry()); } @@ -126,6 +125,7 @@ AU.addRequired(); AU.addRequired(); AU.addRequired(); + AU.addRequired(); AU.setPreservesCFG(); } }; @@ -137,25 +137,28 @@ INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass) +INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) INITIALIZE_PASS_END(LoadStoreVectorizer, DEBUG_TYPE, "Vectorize load and store instructions", false, false); char LoadStoreVectorizer::ID = 0; -Pass *llvm::createLoadStoreVectorizerPass(unsigned VecRegSize) { - return new LoadStoreVectorizer(VecRegSize); +Pass *llvm::createLoadStoreVectorizerPass() { + return new LoadStoreVectorizer(); } bool LoadStoreVectorizer::runOnFunction(Function &F) { AliasAnalysis &AA = getAnalysis().getAAResults(); DominatorTree &DT = getAnalysis().getDomTree(); ScalarEvolution &SE = getAnalysis().getSE(); + TargetTransformInfo &TTI + = getAnalysis().getTTI(F); // Don't vectorize when the attribute NoImplicitFloat is used. if (F.hasFnAttribute(Attribute::NoImplicitFloat)) return false; - Vectorizer V(F, AA, DT, SE, VecRegSize); + Vectorizer V(F, AA, DT, SE, TTI); return V.run(); } @@ -463,6 +466,9 @@ if (!VectorType::isValidElementType(Ty->getScalarType())) continue; + Value *Ptr = LI->getPointerOperand(); + unsigned AS = Ptr->getType()->getPointerAddressSpace(); + unsigned VecRegSize = TTI.getLoadStoreVecRegBitWidth(AS); // No point in looking at these if they're too big to vectorize. if (DL.getTypeSizeInBits(Ty) > VecRegSize / 2) continue; @@ -479,8 +485,8 @@ // TODO: Target hook to filter types. // Save the load locations. - Value *Ptr = GetUnderlyingObject(LI->getPointerOperand(), DL); - LoadRefs[Ptr].push_back(LI); + Value *ObjPtr = GetUnderlyingObject(Ptr, DL); + LoadRefs[ObjPtr].push_back(LI); } else if (StoreInst *SI = dyn_cast(it)) { @@ -491,6 +497,9 @@ if (!VectorType::isValidElementType(Ty->getScalarType())) continue; + Value *Ptr = SI->getPointerOperand(); + unsigned AS = Ptr->getType()->getPointerAddressSpace(); + unsigned VecRegSize = TTI.getLoadStoreVecRegBitWidth(AS); if (DL.getTypeSizeInBits(Ty) > VecRegSize / 2) continue; @@ -506,8 +515,8 @@ // unsupported types or such. // Save store location. - Value *Ptr = GetUnderlyingObject(SI->getPointerOperand(), DL); - StoreRefs[Ptr].push_back(SI); + Value *ObjPtr = GetUnderlyingObject(Ptr, DL); + StoreRefs[ObjPtr].push_back(SI); } } } @@ -599,6 +608,8 @@ StoreInst *S0 = cast(Chain[0]); Type *StoreTy = S0->getValueOperand()->getType(); unsigned Sz = DL.getTypeSizeInBits(StoreTy); + unsigned AS = S0->getPointerAddressSpace(); + unsigned VecRegSize = TTI.getLoadStoreVecRegBitWidth(AS); unsigned VF = VecRegSize / Sz; unsigned ChainSize = Chain.size(); @@ -672,7 +683,6 @@ // Set insert point. Builder.SetInsertPoint(&*Last); - unsigned AS = S0->getPointerAddressSpace(); Value *Vec = UndefValue::get(VecTy); @@ -718,6 +728,8 @@ LoadInst *L0 = cast(Chain[0]); Type *LoadTy = L0->getType(); unsigned Sz = DL.getTypeSizeInBits(LoadTy); + unsigned AS = L0->getPointerAddressSpace(); + unsigned VecRegSize = TTI.getLoadStoreVecRegBitWidth(AS); unsigned VF = VecRegSize / Sz; unsigned ChainSize = Chain.size(); @@ -792,7 +804,6 @@ // Set insert point. Builder.SetInsertPoint(&*Last); - unsigned AS = L0->getPointerAddressSpace(); Value *Bitcast = Builder.CreateBitCast(L0->getPointerOperand(), VecTy->getPointerTo(AS)); Index: test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll =================================================================== --- /dev/null +++ test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll @@ -0,0 +1,51 @@ +; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-4 -load-store-vectorizer -S -o - %s | FileCheck -check-prefix=ELT4 -check-prefix=ALL %s +; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-8 -load-store-vectorizer -S -o - %s | FileCheck -check-prefix=ELT8 -check-prefix=ALL %s +; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-16 -load-store-vectorizer -S -o - %s | FileCheck -check-prefix=ELT16 -check-prefix=ALL %s + +; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32 +; ELT4: store i32 +; ELT4: store i32 +; ELT4: store i32 +; ELT4: store i32 + +; ELT8: store <2 x i32> +; ELT8: store <2 x i32> + +; ELT16: store <4 x i32> +define void @merge_private_store_4_vector_elts_loads_v4i32(i32* %out) #0 { + %out.gep.1 = getelementptr i32, i32* %out, i32 1 + %out.gep.2 = getelementptr i32, i32* %out, i32 2 + %out.gep.3 = getelementptr i32, i32* %out, i32 3 + + store i32 9, i32* %out + store i32 1, i32* %out.gep.1 + store i32 23, i32* %out.gep.2 + store i32 19, i32* %out.gep.3 + ret void +} + +; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i8( +; ALL: store <4 x i8> +define void @merge_private_store_4_vector_elts_loads_v4i8(i8* %out) #0 { + %out.gep.1 = getelementptr i8, i8* %out, i32 1 + %out.gep.2 = getelementptr i8, i8* %out, i32 2 + %out.gep.3 = getelementptr i8, i8* %out, i32 3 + + store i8 9, i8* %out + store i8 1, i8* %out.gep.1 + store i8 23, i8* %out.gep.2 + store i8 19, i8* %out.gep.3 + ret void +} + +; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v2i16( +; ALL: store <2 x i16> +define void @merge_private_store_4_vector_elts_loads_v2i16(i16* %out) #0 { + %out.gep.1 = getelementptr i16, i16* %out, i32 1 + + store i16 9, i16* %out + store i16 12, i16* %out.gep.1 + ret void +} + +attributes #0 = { nounwind } Index: test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll =================================================================== --- test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll +++ test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll @@ -502,7 +502,8 @@ } ; CHECK-LABEL: @merge_local_store_4_constants_i32 -; CHECK: store <4 x i32> , <4 x i32> addrspace(3)* +; CHECK: store <2 x i32> , <2 x i32> addrspace(3)* +; CHECK: store <2 x i32> , <2 x i32> addrspace(3)* define void @merge_local_store_4_constants_i32(i32 addrspace(3)* %out) #0 { %out.gep.1 = getelementptr i32, i32 addrspace(3)* %out, i32 1 %out.gep.2 = getelementptr i32, i32 addrspace(3)* %out, i32 2