Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -566,6 +566,8 @@ /// element. virtual Value *getBroadcastInstrs(Value *V); + LoopVectorizationLegality *getLegal() const { return Legal; } + protected: friend class LoopVectorizationPlanner; @@ -7485,8 +7487,6 @@ Cost = getGatherScatterCost(&I, VF); setWideningDecision(&I, VF, CM_GatherScatter, Cost); } else { - assert((isa(&I) || !VF.isScalable()) && - "Cannot yet scalarize uniform stores"); Cost = getUniformMemOpCost(&I, VF); setWideningDecision(&I, VF, CM_Scalarize, Cost); } @@ -9829,6 +9829,17 @@ return; } + // If the instruction is a store to a uniform address, we only need to + // generate the last lane for the last UF part. + Instruction *I = getUnderlyingInstr(); + if (!IsUniform && isa(I) && + State.ILV->getLegal()->isUniformMemOp(*I)) { + VPLane Lane = VPLane::getLastLaneForVF(State.VF); + State.ILV->scalarizeInstruction( + I, this, *this, VPIteration(State.UF - 1, Lane), IsPredicated, State); + return; + } + // Generate scalar instances for all VF lanes of all UF parts, unless the // instruction is uniform inwhich case generate only the first lane for each // of the UF parts. @@ -9837,9 +9848,8 @@ "Can't scalarize a scalable vector"); for (unsigned Part = 0; Part < State.UF; ++Part) for (unsigned Lane = 0; Lane < EndLane; ++Lane) - State.ILV->scalarizeInstruction(getUnderlyingInstr(), this, *this, - VPIteration(Part, Lane), IsPredicated, - State); + State.ILV->scalarizeInstruction(I, this, *this, VPIteration(Part, Lane), + IsPredicated, State); } void VPBranchOnMaskRecipe::execute(VPTransformState &State) { Index: llvm/test/Transforms/LoopVectorize/AArch64/sve-uniform-store.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LoopVectorize/AArch64/sve-uniform-store.ll @@ -0,0 +1,122 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -loop-vectorize -scalable-vectorization=preferred -mattr=+sve -S < %s | FileCheck %s +target triple = "aarch64-unknown-linux-gnu" + +define void @uniform_store_i1(i1* noalias %dst, i64* noalias %start, i64 %N) { +; CHECK-LABEL: @uniform_store_i1( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]] +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 4 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]] +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]] +; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i64, i64* [[START:%.*]], i64 [[N_VEC]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64* [[START]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT10:%.*]] = insertelement poison, i64* [[START]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT11:%.*]] = shufflevector [[BROADCAST_SPLATINSERT10]], poison, zeroinitializer +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv2i64() +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[INDEX]], i32 0 +; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer +; CHECK-NEXT: [[TMP6:%.*]] = add shufflevector ( insertelement ( poison, i64 0, i32 0), poison, zeroinitializer), [[TMP5]] +; CHECK-NEXT: [[TMP7:%.*]] = add [[DOTSPLAT]], [[TMP6]] +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i64, i64* [[START]], [[TMP7]] +; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i64, i64* [[START]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i64, i64* [[START]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 +; CHECK-NEXT: [[DOTSPLATINSERT4:%.*]] = insertelement poison, i64 [[TMP11]], i32 0 +; CHECK-NEXT: [[DOTSPLAT5:%.*]] = shufflevector [[DOTSPLATINSERT4]], poison, zeroinitializer +; CHECK-NEXT: [[TMP12:%.*]] = add [[DOTSPLAT5]], [[TMP5]] +; CHECK-NEXT: [[TMP13:%.*]] = add [[DOTSPLAT]], [[TMP12]] +; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i64, i64* [[START]], [[TMP13]] +; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP11]], 0 +; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP14]] +; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i64, i64* [[START]], i64 [[TMP15]] +; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[TMP11]], 1 +; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[INDEX]], [[TMP16]] +; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i64, i64* [[START]], i64 [[TMP17]] +; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 2 +; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[TMP20]], 0 +; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 1 +; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[INDEX]], [[TMP22]] +; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i64, i64* [[NEXT_GEP2]], i32 0 +; CHECK-NEXT: [[TMP25:%.*]] = bitcast i64* [[TMP24]] to * +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , * [[TMP25]], align 4 +; CHECK-NEXT: [[TMP26:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP27:%.*]] = mul i32 [[TMP26]], 2 +; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i64, i64* [[NEXT_GEP2]], i32 [[TMP27]] +; CHECK-NEXT: [[TMP29:%.*]] = bitcast i64* [[TMP28]] to * +; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load , * [[TMP29]], align 4 +; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i64, [[NEXT_GEP]], i64 1 +; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds i64, [[NEXT_GEP6]], i64 1 +; CHECK-NEXT: [[TMP32:%.*]] = icmp eq [[TMP30]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP33:%.*]] = icmp eq [[TMP31]], [[BROADCAST_SPLAT11]] +; CHECK-NEXT: [[TMP34:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP35:%.*]] = mul i32 [[TMP34]], 2 +; CHECK-NEXT: [[TMP36:%.*]] = sub i32 [[TMP35]], 1 +; CHECK-NEXT: [[TMP37:%.*]] = extractelement [[TMP33]], i32 [[TMP36]] +; CHECK-NEXT: store i1 [[TMP37]], i1* [[DST:%.*]], align 1 +; CHECK-NEXT: [[TMP38:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP39:%.*]] = mul i64 [[TMP38]], 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP39]] +; CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP40]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; +entry: + br label %for.body + +for.body: + %first.sroa = phi i64* [ %incdec.ptr, %for.body ], [ %start, %entry ] + %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ] + %iv.next = add i64 %iv, 1 + %0 = load i64, i64* %first.sroa + %incdec.ptr = getelementptr inbounds i64, i64* %first.sroa, i64 1 + %cmp.not = icmp eq i64* %incdec.ptr, %start + store i1 %cmp.not, i1* %dst + %cmp = icmp ult i64 %iv, %N + br i1 %cmp, label %for.body, label %end + +end: + ret void +} + +; Ensure conditional i1 stores do not vectorize +define void @cond_store_i1(i1* noalias %dst, i8* noalias %start, i32 %cond, i64 %N) { +; CHECK-LABEL: @cond_store_i1( +; CHECK-NOT: vector.body +; +entry: + br label %for.body + +for.body: + %first.sroa = phi i8* [ %incdec.ptr, %if.end ], [ null, %entry ] + %incdec.ptr = getelementptr inbounds i8, i8* %first.sroa, i64 1 + %0 = load i8, i8* %incdec.ptr + %tobool.not = icmp eq i8 %0, 10 + br i1 %tobool.not, label %if.end, label %if.then + +if.then: + %cmp.store = icmp eq i8* %start, %incdec.ptr + store i1 %cmp.store, i1* %dst + br label %if.end + +if.end: + %cmp.not = icmp eq i8* %incdec.ptr, %start + br i1 %cmp.not, label %for.end, label %for.body + +for.end: + ret void +} Index: llvm/test/Transforms/LoopVectorize/X86/cost-model-assert.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/X86/cost-model-assert.ll +++ llvm/test/Transforms/LoopVectorize/X86/cost-model-assert.ll @@ -62,25 +62,11 @@ ; CHECK-NEXT: [[TMP23:%.*]] = or <4 x i32> [[TMP19]], zeroinitializer ; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i32> [[TMP22]], zeroinitializer ; CHECK-NEXT: [[TMP25:%.*]] = or <4 x i32> [[TMP23]], zeroinitializer -; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i32> [[TMP24]], i32 0 +; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i32> [[TMP25]], i32 3 ; CHECK-NEXT: store i32 [[TMP26]], i32* undef, align 4, !tbaa [[TBAA4:![0-9]+]] -; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i32> [[TMP24]], i32 1 -; CHECK-NEXT: store i32 [[TMP27]], i32* undef, align 4, !tbaa [[TBAA4]] -; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i32> [[TMP24]], i32 2 -; CHECK-NEXT: store i32 [[TMP28]], i32* undef, align 4, !tbaa [[TBAA4]] -; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[TMP24]], i32 3 -; CHECK-NEXT: store i32 [[TMP29]], i32* undef, align 4, !tbaa [[TBAA4]] -; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP25]], i32 0 -; CHECK-NEXT: store i32 [[TMP30]], i32* undef, align 4, !tbaa [[TBAA4]] -; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i32> [[TMP25]], i32 1 -; CHECK-NEXT: store i32 [[TMP31]], i32* undef, align 4, !tbaa [[TBAA4]] -; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i32> [[TMP25]], i32 2 -; CHECK-NEXT: store i32 [[TMP32]], i32* undef, align 4, !tbaa [[TBAA4]] -; CHECK-NEXT: [[TMP33:%.*]] = extractelement <4 x i32> [[TMP25]], i32 3 -; CHECK-NEXT: store i32 [[TMP33]], i32* undef, align 4, !tbaa [[TBAA4]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 -; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP34]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[SW_EPILOG:%.*]], label [[SCALAR_PH]] @@ -91,11 +77,11 @@ ; CHECK-NEXT: [[P_359:%.*]] = phi i8* [ [[ADD_PTR86:%.*]], [[FOR_BODY68]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] ; CHECK-NEXT: [[CONV70:%.*]] = zext i8 [[X]] to i32 ; CHECK-NEXT: [[SHL71:%.*]] = shl nuw i32 [[CONV70]], 24 -; CHECK-NEXT: [[TMP35:%.*]] = load i8, i8* [[P]], align 1, !tbaa [[TBAA1]] -; CHECK-NEXT: [[CONV73:%.*]] = zext i8 [[TMP35]] to i32 +; CHECK-NEXT: [[TMP28:%.*]] = load i8, i8* [[P]], align 1, !tbaa [[TBAA1]] +; CHECK-NEXT: [[CONV73:%.*]] = zext i8 [[TMP28]] to i32 ; CHECK-NEXT: [[SHL74:%.*]] = shl nuw nsw i32 [[CONV73]], 16 ; CHECK-NEXT: [[OR75:%.*]] = or i32 [[SHL74]], [[SHL71]] -; CHECK-NEXT: [[TMP36:%.*]] = load i8, i8* undef, align 1, !tbaa [[TBAA1]] +; CHECK-NEXT: [[TMP29:%.*]] = load i8, i8* undef, align 1, !tbaa [[TBAA1]] ; CHECK-NEXT: [[SHL78:%.*]] = shl nuw nsw i32 undef, 8 ; CHECK-NEXT: [[OR79:%.*]] = or i32 [[OR75]], [[SHL78]] ; CHECK-NEXT: [[CONV81:%.*]] = zext i8 undef to i32 Index: llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll +++ llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll @@ -84,17 +84,11 @@ ; CHECK-NEXT: [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>* ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP22]], align 4 ; CHECK-NEXT: [[TMP23:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], -; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[TMP23]], i32 0 +; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[TMP23]], i32 3 ; CHECK-NEXT: store i32 [[TMP24]], i32* [[ARRAYIDX7_US]], align 4, !llvm.mem.parallel_loop_access !0 -; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i32> [[TMP23]], i32 1 -; CHECK-NEXT: store i32 [[TMP25]], i32* [[ARRAYIDX7_US]], align 4, !llvm.mem.parallel_loop_access !0 -; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i32> [[TMP23]], i32 2 -; CHECK-NEXT: store i32 [[TMP26]], i32* [[ARRAYIDX7_US]], align 4, !llvm.mem.parallel_loop_access !0 -; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i32> [[TMP23]], i32 3 -; CHECK-NEXT: store i32 [[TMP27]], i32* [[ARRAYIDX7_US]], align 4, !llvm.mem.parallel_loop_access !0 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END_US]], label [[SCALAR_PH]] Index: llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll +++ llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll @@ -207,21 +207,6 @@ ; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 8 ; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 12 ; CHECK-NEXT: store i32 0, i32* [[ADDR:%.*]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 -; CHECK-NEXT: store i32 0, i32* [[ADDR]], align 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096 ; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -286,43 +271,13 @@ ; CHECK-NEXT: [[STEP_ADD5:%.*]] = add <4 x i32> [[VEC_IND4]], ; CHECK-NEXT: [[STEP_ADD6:%.*]] = add <4 x i32> [[STEP_ADD5]], ; CHECK-NEXT: [[STEP_ADD7:%.*]] = add <4 x i32> [[STEP_ADD6]], -; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i32> [[VEC_IND4]], i32 0 +; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i32> [[STEP_ADD7]], i32 3 ; CHECK-NEXT: store i32 [[TMP16]], i32* [[ADDR:%.*]], align 4 -; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i32> [[VEC_IND4]], i32 1 -; CHECK-NEXT: store i32 [[TMP17]], i32* [[ADDR]], align 4 -; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i32> [[VEC_IND4]], i32 2 -; CHECK-NEXT: store i32 [[TMP18]], i32* [[ADDR]], align 4 -; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i32> [[VEC_IND4]], i32 3 -; CHECK-NEXT: store i32 [[TMP19]], i32* [[ADDR]], align 4 -; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i32> [[STEP_ADD5]], i32 0 -; CHECK-NEXT: store i32 [[TMP20]], i32* [[ADDR]], align 4 -; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i32> [[STEP_ADD5]], i32 1 -; CHECK-NEXT: store i32 [[TMP21]], i32* [[ADDR]], align 4 -; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i32> [[STEP_ADD5]], i32 2 -; CHECK-NEXT: store i32 [[TMP22]], i32* [[ADDR]], align 4 -; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i32> [[STEP_ADD5]], i32 3 -; CHECK-NEXT: store i32 [[TMP23]], i32* [[ADDR]], align 4 -; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[STEP_ADD6]], i32 0 -; CHECK-NEXT: store i32 [[TMP24]], i32* [[ADDR]], align 4 -; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i32> [[STEP_ADD6]], i32 1 -; CHECK-NEXT: store i32 [[TMP25]], i32* [[ADDR]], align 4 -; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i32> [[STEP_ADD6]], i32 2 -; CHECK-NEXT: store i32 [[TMP26]], i32* [[ADDR]], align 4 -; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i32> [[STEP_ADD6]], i32 3 -; CHECK-NEXT: store i32 [[TMP27]], i32* [[ADDR]], align 4 -; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i32> [[STEP_ADD7]], i32 0 -; CHECK-NEXT: store i32 [[TMP28]], i32* [[ADDR]], align 4 -; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[STEP_ADD7]], i32 1 -; CHECK-NEXT: store i32 [[TMP29]], i32* [[ADDR]], align 4 -; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[STEP_ADD7]], i32 2 -; CHECK-NEXT: store i32 [[TMP30]], i32* [[ADDR]], align 4 -; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i32> [[STEP_ADD7]], i32 3 -; CHECK-NEXT: store i32 [[TMP31]], i32* [[ADDR]], align 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD2]], ; CHECK-NEXT: [[VEC_IND_NEXT9]] = add <4 x i32> [[STEP_ADD7]], -; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096 -; CHECK-NEXT: br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096 +; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 4097, 4096 ; CHECK-NEXT: br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]] @@ -412,21 +367,6 @@ ; CHECK-NEXT: [[TMP5:%.*]] = load i32, i32* [[A]], align 4, !alias.scope !12 ; CHECK-NEXT: [[TMP6:%.*]] = load i32, i32* [[A]], align 4, !alias.scope !12 ; CHECK-NEXT: [[TMP7:%.*]] = load i32, i32* [[A]], align 4, !alias.scope !12 -; CHECK-NEXT: store i32 [[TMP4]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 -; CHECK-NEXT: store i32 [[TMP4]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 -; CHECK-NEXT: store i32 [[TMP4]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 -; CHECK-NEXT: store i32 [[TMP4]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 -; CHECK-NEXT: store i32 [[TMP5]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 -; CHECK-NEXT: store i32 [[TMP5]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 -; CHECK-NEXT: store i32 [[TMP5]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 -; CHECK-NEXT: store i32 [[TMP5]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 -; CHECK-NEXT: store i32 [[TMP6]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 -; CHECK-NEXT: store i32 [[TMP6]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 -; CHECK-NEXT: store i32 [[TMP6]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 -; CHECK-NEXT: store i32 [[TMP6]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 -; CHECK-NEXT: store i32 [[TMP7]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 -; CHECK-NEXT: store i32 [[TMP7]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 -; CHECK-NEXT: store i32 [[TMP7]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 ; CHECK-NEXT: store i32 [[TMP7]], i32* [[B]], align 4, !alias.scope !15, !noalias !12 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096 Index: llvm/test/Transforms/LoopVectorize/pr44488-predication.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/pr44488-predication.ll +++ llvm/test/Transforms/LoopVectorize/pr44488-predication.ll @@ -45,13 +45,11 @@ ; CHECK: pred.srem.continue4: ; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i16> [ [[TMP9]], [[PRED_SREM_CONTINUE]] ], [ [[TMP12]], [[PRED_SREM_IF3]] ] ; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i16> , <2 x i16> [[TMP13]] -; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i16> [[PREDPHI]], i32 0 +; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i16> [[PREDPHI]], i32 1 ; CHECK-NEXT: store i16 [[TMP14]], i16* @v_39, align 1 -; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i16> [[PREDPHI]], i32 1 -; CHECK-NEXT: store i16 [[TMP15]], i16* @v_39, align 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 -; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12 -; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]] +; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12 +; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 12, 12 ; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] @@ -74,7 +72,7 @@ ; CHECK-NEXT: store i16 [[COND6]], i16* @v_39, align 1 ; CHECK-NEXT: [[INC7]] = add nsw i16 [[I_07]], 1 ; CHECK-NEXT: [[CMP:%.*]] = icmp slt i16 [[INC7]], 111 -; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], [[LOOP2:!llvm.loop !.*]] +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP2:![0-9]+]] ; CHECK: exit: ; CHECK-NEXT: [[RV:%.*]] = load i16, i16* @v_39, align 1 ; CHECK-NEXT: ret i16 [[RV]] Index: llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll +++ llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll @@ -48,8 +48,6 @@ ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 0 ; CHECK-NEXT: store i32 0, i32* @f.e, align 1, !alias.scope !0, !noalias !3 -; CHECK-NEXT: store i32 0, i32* @f.e, align 1, !alias.scope !0, !noalias !3 -; CHECK-NEXT: store i8 10, i8* [[TMP0]], align 1 ; CHECK-NEXT: store i8 10, i8* [[TMP0]], align 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 ; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 500 Index: llvm/test/Transforms/LoopVectorize/skeleton-lcssa-crash.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/skeleton-lcssa-crash.ll +++ llvm/test/Transforms/LoopVectorize/skeleton-lcssa-crash.ll @@ -55,13 +55,11 @@ ; CHECK-NEXT: [[TMP6:%.*]] = bitcast i16* [[TMP5]] to <2 x i16>* ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i16>, <2 x i16>* [[TMP6]], align 2, !alias.scope !0 ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, i16* [[L_2]], i64 0 -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i16> [[WIDE_LOAD]], i32 0 +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i16> [[WIDE_LOAD]], i32 1 ; CHECK-NEXT: store i16 [[TMP8]], i16* [[TMP7]], align 2, !alias.scope !3, !noalias !0 -; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i16> [[WIDE_LOAD]], i32 1 -; CHECK-NEXT: store i16 [[TMP9]], i16* [[TMP7]], align 2, !alias.scope !3, !noalias !0 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] Index: llvm/test/Transforms/LoopVectorize/uniform-store.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LoopVectorize/uniform-store.ll @@ -0,0 +1,68 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -loop-vectorize -S < %s | FileCheck %s +target triple = "aarch64-unknown-linux-gnu" + +define void @uniform_store_i1(i1* noalias %dst, i64* noalias %start, i64 %N) { +; CHECK-LABEL: @uniform_store_i1( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]] +; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i64, i64* [[START:%.*]], i64 [[N_VEC]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64*> poison, i64* [[START]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64*> [[BROADCAST_SPLATINSERT]], <2 x i64*> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <2 x i64*> poison, i64* [[START]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <2 x i64*> [[BROADCAST_SPLATINSERT6]], <2 x i64*> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i64, i64* [[START]], i64 [[TMP1]] +; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i64, i64* [[START]], i64 [[TMP2]] +; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i64*> poison, i64* [[NEXT_GEP]], i32 0 +; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i64*> [[TMP3]], i64* [[NEXT_GEP2]], i32 1 +; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 2 +; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i64, i64* [[START]], i64 [[TMP5]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 3 +; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i64, i64* [[START]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i64*> poison, i64* [[NEXT_GEP3]], i32 0 +; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i64*> [[TMP7]], i64* [[NEXT_GEP4]], i32 1 +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i64, i64* [[NEXT_GEP]], i32 0 +; CHECK-NEXT: [[TMP12:%.*]] = bitcast i64* [[TMP11]] to <2 x i64>* +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, <2 x i64>* [[TMP12]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i64, i64* [[NEXT_GEP]], i32 2 +; CHECK-NEXT: [[TMP14:%.*]] = bitcast i64* [[TMP13]] to <2 x i64>* +; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <2 x i64>, <2 x i64>* [[TMP14]], align 4 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, <2 x i64*> [[TMP4]], i64 1 +; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, <2 x i64*> [[TMP8]], i64 1 +; CHECK-NEXT: [[TMP17:%.*]] = icmp eq <2 x i64*> [[TMP15]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP18:%.*]] = icmp eq <2 x i64*> [[TMP16]], [[BROADCAST_SPLAT7]] +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x i1> [[TMP18]], i32 1 +; CHECK-NEXT: store i1 [[TMP19]], i1* [[DST:%.*]], align 1 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; +entry: + br label %for.body + +for.body: + %first.sroa = phi i64* [ %incdec.ptr, %for.body ], [ %start, %entry ] + %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ] + %iv.next = add i64 %iv, 1 + %0 = load i64, i64* %first.sroa + %incdec.ptr = getelementptr inbounds i64, i64* %first.sroa, i64 1 + %cmp.not = icmp eq i64* %incdec.ptr, %start + store i1 %cmp.not, i1* %dst + %cmp = icmp ult i64 %iv, %N + br i1 %cmp, label %for.body, label %end + +end: + ret void +}