Index: lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp +++ lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp @@ -348,9 +348,6 @@ static Value * calculateVectorIndex(Value *Ptr, const std::map &GEPIdx) { - if (isa(Ptr)) - return Constant::getNullValue(Type::getInt32Ty(Ptr->getContext())); - GetElementPtrInst *GEP = cast(Ptr); auto I = GEPIdx.find(GEP); @@ -360,11 +357,11 @@ static Value* GEPToVectorIndex(GetElementPtrInst *GEP) { // FIXME we only support simple cases if (GEP->getNumOperands() != 3) - return NULL; + return nullptr; ConstantInt *I0 = dyn_cast(GEP->getOperand(1)); if (!I0 || !I0->isZero()) - return NULL; + return nullptr; return GEP->getOperand(2); } @@ -398,7 +395,8 @@ // are just being conservative for now. if (!AllocaTy || AllocaTy->getElementType()->isVectorTy() || - AllocaTy->getNumElements() > 4) { + AllocaTy->getNumElements() > 4 || + AllocaTy->getNumElements() < 2) { DEBUG(dbgs() << " Cannot convert type to vector\n"); return false; } @@ -443,9 +441,11 @@ IRBuilder<> Builder(Inst); switch (Inst->getOpcode()) { case Instruction::Load: { + Type *VecPtrTy = VectorTy->getPointerTo(AMDGPUAS::PRIVATE_ADDRESS); Value *Ptr = Inst->getOperand(0); Value *Index = calculateVectorIndex(Ptr, GEPVectorIdx); - Value *BitCast = Builder.CreateBitCast(Alloca, VectorTy->getPointerTo(0)); + + Value *BitCast = Builder.CreateBitCast(Alloca, VecPtrTy); Value *VecValue = Builder.CreateLoad(BitCast); Value *ExtractElement = Builder.CreateExtractElement(VecValue, Index); Inst->replaceAllUsesWith(ExtractElement); @@ -453,9 +453,11 @@ break; } case Instruction::Store: { + Type *VecPtrTy = VectorTy->getPointerTo(AMDGPUAS::PRIVATE_ADDRESS); + Value *Ptr = Inst->getOperand(1); Value *Index = calculateVectorIndex(Ptr, GEPVectorIdx); - Value *BitCast = Builder.CreateBitCast(Alloca, VectorTy->getPointerTo(0)); + Value *BitCast = Builder.CreateBitCast(Alloca, VecPtrTy); Value *VecValue = Builder.CreateLoad(BitCast); Value *NewVecValue = Builder.CreateInsertElement(VecValue, Inst->getOperand(0), @@ -469,7 +471,6 @@ break; default: - Inst->dump(); llvm_unreachable("Inconsistency in instructions promotable to vector"); } } Index: test/CodeGen/AMDGPU/amdgpu.private-memory.ll =================================================================== --- test/CodeGen/AMDGPU/amdgpu.private-memory.ll +++ test/CodeGen/AMDGPU/amdgpu.private-memory.ll @@ -417,12 +417,6 @@ ret void } -; HSAOPT: !0 = !{} -; HSAOPT: !1 = !{i32 0, i32 2048} - -; NOHSAOPT: !0 = !{i32 0, i32 2048} - - ; FUNC-LABEL: v16i32_stack: ; R600: MOVA_INT @@ -527,4 +521,33 @@ ret void } +; OPT-LABEL: @direct_alloca_read_0xi32( +; OPT: store [0 x i32] undef, [0 x i32] addrspace(3)* +; OPT: load [0 x i32], [0 x i32] addrspace(3)* +define void @direct_alloca_read_0xi32([0 x i32] addrspace(1)* %out, i32 %index) { +entry: + %tmp = alloca [0 x i32] + store [0 x i32] [], [0 x i32]* %tmp + %load = load [0 x i32], [0 x i32]* %tmp + store [0 x i32] %load, [0 x i32] addrspace(1)* %out + ret void +} + +; OPT-LABEL: @direct_alloca_read_1xi32( +; OPT: store [1 x i32] zeroinitializer, [1 x i32] addrspace(3)* +; OPT: load [1 x i32], [1 x i32] addrspace(3)* +define void @direct_alloca_read_1xi32([1 x i32] addrspace(1)* %out, i32 %index) { +entry: + %tmp = alloca [1 x i32] + store [1 x i32] [i32 0], [1 x i32]* %tmp + %load = load [1 x i32], [1 x i32]* %tmp + store [1 x i32] %load, [1 x i32] addrspace(1)* %out + ret void +} + attributes #0 = { nounwind "amdgpu-max-waves-per-eu"="2" } + +; HSAOPT: !0 = !{} +; HSAOPT: !1 = !{i32 0, i32 2048} + +; NOHSAOPT: !0 = !{i32 0, i32 2048} Index: test/CodeGen/AMDGPU/vector-alloca.ll =================================================================== --- test/CodeGen/AMDGPU/vector-alloca.ll +++ test/CodeGen/AMDGPU/vector-alloca.ll @@ -3,6 +3,11 @@ ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=+promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck --check-prefix=EG -check-prefix=FUNC %s +; RUN: opt -S -mtriple=amdgcn-- -amdgpu-promote-alloca -sroa -instcombine < %s | FileCheck -check-prefix=OPT %s + +; OPT-LABEL: @vector_read( +; OPT: %0 = extractelement <4 x i32> , i32 %index +; OPT: store i32 %0, i32 addrspace(1)* %out, align 4 ; FUNC-LABEL: {{^}}vector_read: ; EG: MOV @@ -12,21 +17,26 @@ ; EG: MOVA_INT define void @vector_read(i32 addrspace(1)* %out, i32 %index) { entry: - %0 = alloca [4 x i32] - %x = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 0 - %y = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 1 - %z = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 2 - %w = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 3 + %tmp = alloca [4 x i32] + %x = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 0 + %y = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 1 + %z = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 2 + %w = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 3 store i32 0, i32* %x store i32 1, i32* %y store i32 2, i32* %z store i32 3, i32* %w - %1 = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 %index - %2 = load i32, i32* %1 - store i32 %2, i32 addrspace(1)* %out + %tmp1 = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 %index + %tmp2 = load i32, i32* %tmp1 + store i32 %tmp2, i32 addrspace(1)* %out ret void } +; OPT-LABEL: @vector_write( +; OPT: %0 = insertelement <4 x i32> zeroinitializer, i32 1, i32 %w_index +; OPT: %1 = extractelement <4 x i32> %0, i32 %r_index +; OPT: store i32 %1, i32 addrspace(1)* %out, align 4 + ; FUNC-LABEL: {{^}}vector_write: ; EG: MOV ; EG: MOV @@ -36,42 +46,95 @@ ; EG: MOVA_INT define void @vector_write(i32 addrspace(1)* %out, i32 %w_index, i32 %r_index) { entry: - %0 = alloca [4 x i32] - %x = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 0 - %y = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 1 - %z = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 2 - %w = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 3 + %tmp = alloca [4 x i32] + %x = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 0 + %y = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 1 + %z = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 2 + %w = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 3 store i32 0, i32* %x store i32 0, i32* %y store i32 0, i32* %z store i32 0, i32* %w - %1 = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 %w_index - store i32 1, i32* %1 - %2 = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 %r_index - %3 = load i32, i32* %2 - store i32 %3, i32 addrspace(1)* %out + %tmp1 = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 %w_index + store i32 1, i32* %tmp1 + %tmp2 = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 %r_index + %tmp3 = load i32, i32* %tmp2 + store i32 %tmp3, i32 addrspace(1)* %out ret void } ; This test should be optimize to: ; store i32 0, i32 addrspace(1)* %out + +; OPT-LABEL: @bitcast_gep( +; OPT-LABEL: store i32 0, i32 addrspace(1)* %out, align 4 + ; FUNC-LABEL: {{^}}bitcast_gep: ; EG: STORE_RAW define void @bitcast_gep(i32 addrspace(1)* %out, i32 %w_index, i32 %r_index) { entry: - %0 = alloca [4 x i32] - %x = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 0 - %y = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 1 - %z = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 2 - %w = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 3 + %tmp = alloca [4 x i32] + %x = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 0 + %y = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 1 + %z = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 2 + %w = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 3 store i32 0, i32* %x store i32 0, i32* %y store i32 0, i32* %z store i32 0, i32* %w - %1 = getelementptr [4 x i32], [4 x i32]* %0, i32 0, i32 1 - %2 = bitcast i32* %1 to [4 x i32]* - %3 = getelementptr [4 x i32], [4 x i32]* %2, i32 0, i32 0 - %4 = load i32, i32* %3 - store i32 %4, i32 addrspace(1)* %out + %tmp1 = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 1 + %tmp2 = bitcast i32* %tmp1 to [4 x i32]* + %tmp3 = getelementptr [4 x i32], [4 x i32]* %tmp2, i32 0, i32 0 + %tmp4 = load i32, i32* %tmp3 + store i32 %tmp4, i32 addrspace(1)* %out + ret void +} + +; OPT-LABEL: @vector_read_bitcast_gep( +; OPT: %0 = extractelement <4 x i32> , i32 %index +; OPT: store i32 %0, i32 addrspace(1)* %out, align 4 +define void @vector_read_bitcast_gep(i32 addrspace(1)* %out, i32 %index) { +entry: + %tmp = alloca [4 x i32] + %x = getelementptr inbounds [4 x i32], [4 x i32]* %tmp, i32 0, i32 0 + %y = getelementptr inbounds [4 x i32], [4 x i32]* %tmp, i32 0, i32 1 + %z = getelementptr inbounds [4 x i32], [4 x i32]* %tmp, i32 0, i32 2 + %w = getelementptr inbounds [4 x i32], [4 x i32]* %tmp, i32 0, i32 3 + %bc = bitcast i32* %x to float* + store float 1.0, float* %bc + store i32 1, i32* %y + store i32 2, i32* %z + store i32 3, i32* %w + %tmp1 = getelementptr inbounds [4 x i32], [4 x i32]* %tmp, i32 0, i32 %index + %tmp2 = load i32, i32* %tmp1 + store i32 %tmp2, i32 addrspace(1)* %out + ret void +} + +; FIXME: Should be able to promote this. Instcombine should fold the +; cast in the hasOneUse case so it might not matter in practice + +; OPT-LABEL: @vector_read_bitcast_alloca( +; OPT: alloca [4 x float] +; OPT: store float +; OPT: store float +; OPT: store float +; OPT: store float +; OPT: load float +define void @vector_read_bitcast_alloca(float addrspace(1)* %out, i32 %index) { +entry: + %tmp = alloca [4 x i32] + %tmp.bc = bitcast [4 x i32]* %tmp to [4 x float]* + %x = getelementptr inbounds [4 x float], [4 x float]* %tmp.bc, i32 0, i32 0 + %y = getelementptr inbounds [4 x float], [4 x float]* %tmp.bc, i32 0, i32 1 + %z = getelementptr inbounds [4 x float], [4 x float]* %tmp.bc, i32 0, i32 2 + %w = getelementptr inbounds [4 x float], [4 x float]* %tmp.bc, i32 0, i32 3 + store float 0.0, float* %x + store float 1.0, float* %y + store float 2.0, float* %z + store float 4.0, float* %w + %tmp1 = getelementptr inbounds [4 x float], [4 x float]* %tmp.bc, i32 0, i32 %index + %tmp2 = load float, float* %tmp1 + store float %tmp2, float addrspace(1)* %out ret void }