diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -1235,6 +1235,11 @@ /// split during legalization. Zero is returned when the answer is unknown. unsigned getNumberOfParts(Type *Tp) const; + /// \returns Whether the LV should set the cost of + /// predicated scalarized memory operations to be artificially large, + /// to effectively prevent their vectorization. + bool useEmulatedMaskMemRefHack() const; + /// \returns The cost of the address computation. For most targets this can be /// merged into the instruction indexing mode. Some targets might want to /// distinguish between address computation for memory operations on vector @@ -1698,6 +1703,7 @@ ArrayRef Tys, TTI::TargetCostKind CostKind) = 0; virtual unsigned getNumberOfParts(Type *Tp) = 0; + virtual bool useEmulatedMaskMemRefHack() = 0; virtual InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *SE, const SCEV *Ptr) = 0; virtual InstructionCost @@ -2236,6 +2242,9 @@ unsigned getNumberOfParts(Type *Tp) override { return Impl.getNumberOfParts(Tp); } + bool useEmulatedMaskMemRefHack() override { + return Impl.useEmulatedMaskMemRefHack(); + } InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *SE, const SCEV *Ptr) override { return Impl.getAddressComputationCost(Ty, SE, Ptr); diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -635,6 +635,8 @@ // Assume that we have a register of the right size for the type. unsigned getNumberOfParts(Type *Tp) const { return 1; } + bool useEmulatedMaskMemRefHack() const { return false; } + InstructionCost getAddressComputationCost(Type *Tp, ScalarEvolution *, const SCEV *) const { return 0; diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -2029,6 +2029,10 @@ return LT.first.isValid() ? *LT.first.getValue() : 0; } + bool useEmulatedMaskMemRefHack() { + return true; // FIXME + } + InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *, const SCEV *) { return 0; diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -903,6 +903,10 @@ return TTIImpl->getNumberOfParts(Tp); } +bool TargetTransformInfo::useEmulatedMaskMemRefHack() const { + return TTIImpl->useEmulatedMaskMemRefHack(); +} + InstructionCost TargetTransformInfo::getAddressComputationCost(Type *Tp, ScalarEvolution *SE, const SCEV *Ptr) const { diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -161,6 +161,7 @@ Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I); + bool useEmulatedMaskMemRefHack(); InstructionCost getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr); diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -4055,6 +4055,8 @@ return Cost + LT.first; } +bool X86TTIImpl::useEmulatedMaskMemRefHack() { return !ST->hasAVX2(); } + InstructionCost X86TTIImpl::getAddressComputationCost(Type *Ty, ScalarEvolution *SE, const SCEV *Ptr) { diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -6707,6 +6707,8 @@ // Limited number of Masked Store/Scatter emulation was allowed. assert(isPredicatedInst(I) && "Expecting a scalar emulated instruction"); + if (!TTI.useEmulatedMaskMemRefHack()) + return false; return isa(I) || (isa(I) && NumPredStores > NumberOfStoresToPredicate); diff --git a/llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll --- a/llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll +++ b/llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll @@ -36,11 +36,11 @@ ; AVX1: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i32, i32* %inB, align 4 ; ; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i32, i32* %inB, align 4 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i32, i32* %inB, align 4 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i32, i32* %inB, align 4 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i32, i32* %inB, align 4 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i32, i32* %inB, align 4 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i32, i32* %inB, align 4 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 2 for VF 2 For instruction: %valB.loaded = load i32, i32* %inB, align 4 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 4 for VF 4 For instruction: %valB.loaded = load i32, i32* %inB, align 4 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 9 for VF 8 For instruction: %valB.loaded = load i32, i32* %inB, align 4 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 18 for VF 16 For instruction: %valB.loaded = load i32, i32* %inB, align 4 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 36 for VF 32 For instruction: %valB.loaded = load i32, i32* %inB, align 4 ; ; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i32, i32* %inB, align 4 ; AVX2-FASTGATHER: LV: Found an estimated cost of 4 for VF 2 For instruction: %valB.loaded = load i32, i32* %inB, align 4 @@ -50,8 +50,8 @@ ; AVX2-FASTGATHER: LV: Found an estimated cost of 48 for VF 32 For instruction: %valB.loaded = load i32, i32* %inB, align 4 ; ; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i32, i32* %inB, align 4 -; AVX512: LV: Found an estimated cost of 10 for VF 2 For instruction: %valB.loaded = load i32, i32* %inB, align 4 -; AVX512: LV: Found an estimated cost of 22 for VF 4 For instruction: %valB.loaded = load i32, i32* %inB, align 4 +; AVX512: LV: Found an estimated cost of 5 for VF 2 For instruction: %valB.loaded = load i32, i32* %inB, align 4 +; AVX512: LV: Found an estimated cost of 11 for VF 4 For instruction: %valB.loaded = load i32, i32* %inB, align 4 ; AVX512: LV: Found an estimated cost of 10 for VF 8 For instruction: %valB.loaded = load i32, i32* %inB, align 4 ; AVX512: LV: Found an estimated cost of 18 for VF 16 For instruction: %valB.loaded = load i32, i32* %inB, align 4 ; AVX512: LV: Found an estimated cost of 36 for VF 32 For instruction: %valB.loaded = load i32, i32* %inB, align 4 diff --git a/llvm/test/Analysis/CostModel/X86/masked-gather-i64-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/masked-gather-i64-with-i8-index.ll --- a/llvm/test/Analysis/CostModel/X86/masked-gather-i64-with-i8-index.ll +++ b/llvm/test/Analysis/CostModel/X86/masked-gather-i64-with-i8-index.ll @@ -36,11 +36,11 @@ ; AVX1: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i64, i64* %inB, align 8 ; ; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i64, i64* %inB, align 8 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 2 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 5 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 10 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 20 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 40 for VF 32 For instruction: %valB.loaded = load i64, i64* %inB, align 8 ; ; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8 ; AVX2-FASTGATHER: LV: Found an estimated cost of 4 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8 @@ -50,8 +50,8 @@ ; AVX2-FASTGATHER: LV: Found an estimated cost of 48 for VF 32 For instruction: %valB.loaded = load i64, i64* %inB, align 8 ; ; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8 -; AVX512: LV: Found an estimated cost of 10 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8 -; AVX512: LV: Found an estimated cost of 24 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8 +; AVX512: LV: Found an estimated cost of 5 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8 +; AVX512: LV: Found an estimated cost of 12 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8 ; AVX512: LV: Found an estimated cost of 10 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8 ; AVX512: LV: Found an estimated cost of 20 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8 ; AVX512: LV: Found an estimated cost of 40 for VF 32 For instruction: %valB.loaded = load i64, i64* %inB, align 8 diff --git a/llvm/test/Analysis/CostModel/X86/masked-interleaved-load-i16.ll b/llvm/test/Analysis/CostModel/X86/masked-interleaved-load-i16.ll --- a/llvm/test/Analysis/CostModel/X86/masked-interleaved-load-i16.ll +++ b/llvm/test/Analysis/CostModel/X86/masked-interleaved-load-i16.ll @@ -89,30 +89,30 @@ ; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 ; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i4 = load i16, i16* %arrayidx7, align 2 ; -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %i4 = load i16, i16* %arrayidx7, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 2 for VF 2 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 2 for VF 2 For instruction: %i4 = load i16, i16* %arrayidx7, align 2 ; -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %i4 = load i16, i16* %arrayidx7, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 4 for VF 4 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 4 for VF 4 For instruction: %i4 = load i16, i16* %arrayidx7, align 2 ; -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %i4 = load i16, i16* %arrayidx7, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 8 for VF 8 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 8 for VF 8 For instruction: %i4 = load i16, i16* %arrayidx7, align 2 ; -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %i4 = load i16, i16* %arrayidx7, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 17 for VF 16 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 17 for VF 16 For instruction: %i4 = load i16, i16* %arrayidx7, align 2 ; ENABLED_MASKED_STRIDED: LV: Checking a loop in "test2" ; ; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 ; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i4 = load i16, i16* %arrayidx7, align 2 ; -; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 8 for VF 2 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 +; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 2 for VF 2 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 ; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 2 For instruction: %i4 = load i16, i16* %arrayidx7, align 2 ; -; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 11 for VF 4 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 +; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 4 for VF 4 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 ; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 4 For instruction: %i4 = load i16, i16* %arrayidx7, align 2 ; -; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 11 for VF 8 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 +; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 8 for VF 8 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 ; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 8 For instruction: %i4 = load i16, i16* %arrayidx7, align 2 ; ; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 17 for VF 16 For instruction: %i2 = load i16, i16* %arrayidx2, align 2 @@ -164,17 +164,17 @@ ; DISABLED_MASKED_STRIDED: LV: Checking a loop in "test" ; ; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 2 for VF 2 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 4 for VF 4 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 8 for VF 8 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 17 for VF 16 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 ; ENABLED_MASKED_STRIDED: LV: Checking a loop in "test" ; ; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 -; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 7 for VF 2 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 -; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 9 for VF 4 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 -; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 9 for VF 8 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 +; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 2 for VF 2 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 +; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 4 for VF 4 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 +; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 8 for VF 8 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 ; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 14 for VF 16 For instruction: %i4 = load i16, i16* %arrayidx6, align 2 define void @test(i16* noalias nocapture %points, i16* noalias nocapture readonly %x, i16* noalias nocapture readnone %y) { diff --git a/llvm/test/Analysis/CostModel/X86/masked-interleaved-store-i16.ll b/llvm/test/Analysis/CostModel/X86/masked-interleaved-store-i16.ll --- a/llvm/test/Analysis/CostModel/X86/masked-interleaved-store-i16.ll +++ b/llvm/test/Analysis/CostModel/X86/masked-interleaved-store-i16.ll @@ -89,17 +89,17 @@ ; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %0, i16* %arrayidx2, align 2 ; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %2, i16* %arrayidx7, align 2 ; -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 5 for VF 2 For instruction: store i16 %0, i16* %arrayidx2, align 2 -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 2 For instruction: store i16 %2, i16* %arrayidx7, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %0, i16* %arrayidx2, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %2, i16* %arrayidx7, align 2 ; -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 11 for VF 4 For instruction: store i16 %0, i16* %arrayidx2, align 2 -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 4 For instruction: store i16 %2, i16* %arrayidx7, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 4 for VF 4 For instruction: store i16 %0, i16* %arrayidx2, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 4 for VF 4 For instruction: store i16 %2, i16* %arrayidx7, align 2 ; -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 23 for VF 8 For instruction: store i16 %0, i16* %arrayidx2, align 2 -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 8 For instruction: store i16 %2, i16* %arrayidx7, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 8 for VF 8 For instruction: store i16 %0, i16* %arrayidx2, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 8 for VF 8 For instruction: store i16 %2, i16* %arrayidx7, align 2 ; -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 50 for VF 16 For instruction: store i16 %0, i16* %arrayidx2, align 2 -; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 16 For instruction: store i16 %2, i16* %arrayidx7, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 20 for VF 16 For instruction: store i16 %0, i16* %arrayidx2, align 2 +; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 20 for VF 16 For instruction: store i16 %2, i16* %arrayidx7, align 2 ; ENABLED_MASKED_STRIDED: LV: Checking a loop in "test2" ; @@ -107,16 +107,16 @@ ; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %2, i16* %arrayidx7, align 2 ; ; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 2 For instruction: store i16 %0, i16* %arrayidx2, align 2 -; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 10 for VF 2 For instruction: store i16 %2, i16* %arrayidx7, align 2 +; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %2, i16* %arrayidx7, align 2 ; ; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 4 For instruction: store i16 %0, i16* %arrayidx2, align 2 -; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 14 for VF 4 For instruction: store i16 %2, i16* %arrayidx7, align 2 +; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 4 for VF 4 For instruction: store i16 %2, i16* %arrayidx7, align 2 ; ; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 8 For instruction: store i16 %0, i16* %arrayidx2, align 2 -; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 14 for VF 8 For instruction: store i16 %2, i16* %arrayidx7, align 2 +; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 8 for VF 8 For instruction: store i16 %2, i16* %arrayidx7, align 2 ; ; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 16 For instruction: store i16 %0, i16* %arrayidx2, align 2 -; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 27 for VF 16 For instruction: store i16 %2, i16* %arrayidx7, align 2 +; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 20 for VF 16 For instruction: store i16 %2, i16* %arrayidx7, align 2 define void @test2(i16* noalias nocapture %points, i32 %numPoints, i16* noalias nocapture readonly %x, i16* noalias nocapture readonly %y) { entry: diff --git a/llvm/test/Analysis/CostModel/X86/masked-load-i16.ll b/llvm/test/Analysis/CostModel/X86/masked-load-i16.ll --- a/llvm/test/Analysis/CostModel/X86/masked-load-i16.ll +++ b/llvm/test/Analysis/CostModel/X86/masked-load-i16.ll @@ -35,18 +35,18 @@ ; AVX1: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i16, i16* %inB, align 2 ; ; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i16, i16* %inB, align 2 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i16, i16* %inB, align 2 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i16, i16* %inB, align 2 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i16, i16* %inB, align 2 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i16, i16* %inB, align 2 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i16, i16* %inB, align 2 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 2 for VF 2 For instruction: %valB.loaded = load i16, i16* %inB, align 2 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 4 for VF 4 For instruction: %valB.loaded = load i16, i16* %inB, align 2 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 8 for VF 8 For instruction: %valB.loaded = load i16, i16* %inB, align 2 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 17 for VF 16 For instruction: %valB.loaded = load i16, i16* %inB, align 2 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 34 for VF 32 For instruction: %valB.loaded = load i16, i16* %inB, align 2 ; ; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i16, i16* %inB, align 2 -; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i16, i16* %inB, align 2 -; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i16, i16* %inB, align 2 -; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i16, i16* %inB, align 2 -; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i16, i16* %inB, align 2 -; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i16, i16* %inB, align 2 +; AVX2-FASTGATHER: LV: Found an estimated cost of 2 for VF 2 For instruction: %valB.loaded = load i16, i16* %inB, align 2 +; AVX2-FASTGATHER: LV: Found an estimated cost of 4 for VF 4 For instruction: %valB.loaded = load i16, i16* %inB, align 2 +; AVX2-FASTGATHER: LV: Found an estimated cost of 8 for VF 8 For instruction: %valB.loaded = load i16, i16* %inB, align 2 +; AVX2-FASTGATHER: LV: Found an estimated cost of 17 for VF 16 For instruction: %valB.loaded = load i16, i16* %inB, align 2 +; AVX2-FASTGATHER: LV: Found an estimated cost of 34 for VF 32 For instruction: %valB.loaded = load i16, i16* %inB, align 2 ; ; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i16, i16* %inB, align 2 ; AVX512: LV: Found an estimated cost of 2 for VF 2 For instruction: %valB.loaded = load i16, i16* %inB, align 2 diff --git a/llvm/test/Analysis/CostModel/X86/masked-load-i8.ll b/llvm/test/Analysis/CostModel/X86/masked-load-i8.ll --- a/llvm/test/Analysis/CostModel/X86/masked-load-i8.ll +++ b/llvm/test/Analysis/CostModel/X86/masked-load-i8.ll @@ -35,18 +35,18 @@ ; AVX1: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i8, i8* %inB, align 1 ; ; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i8, i8* %inB, align 1 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i8, i8* %inB, align 1 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i8, i8* %inB, align 1 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i8, i8* %inB, align 1 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i8, i8* %inB, align 1 -; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i8, i8* %inB, align 1 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 2 for VF 2 For instruction: %valB.loaded = load i8, i8* %inB, align 1 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 4 for VF 4 For instruction: %valB.loaded = load i8, i8* %inB, align 1 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 8 for VF 8 For instruction: %valB.loaded = load i8, i8* %inB, align 1 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 16 for VF 16 For instruction: %valB.loaded = load i8, i8* %inB, align 1 +; AVX2-SLOWGATHER: LV: Found an estimated cost of 33 for VF 32 For instruction: %valB.loaded = load i8, i8* %inB, align 1 ; ; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i8, i8* %inB, align 1 -; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i8, i8* %inB, align 1 -; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i8, i8* %inB, align 1 -; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i8, i8* %inB, align 1 -; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i8, i8* %inB, align 1 -; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i8, i8* %inB, align 1 +; AVX2-FASTGATHER: LV: Found an estimated cost of 2 for VF 2 For instruction: %valB.loaded = load i8, i8* %inB, align 1 +; AVX2-FASTGATHER: LV: Found an estimated cost of 4 for VF 4 For instruction: %valB.loaded = load i8, i8* %inB, align 1 +; AVX2-FASTGATHER: LV: Found an estimated cost of 8 for VF 8 For instruction: %valB.loaded = load i8, i8* %inB, align 1 +; AVX2-FASTGATHER: LV: Found an estimated cost of 16 for VF 16 For instruction: %valB.loaded = load i8, i8* %inB, align 1 +; AVX2-FASTGATHER: LV: Found an estimated cost of 33 for VF 32 For instruction: %valB.loaded = load i8, i8* %inB, align 1 ; ; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i8, i8* %inB, align 1 ; AVX512: LV: Found an estimated cost of 2 for VF 2 For instruction: %valB.loaded = load i8, i8* %inB, align 1 diff --git a/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll --- a/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll @@ -95,7 +95,7 @@ ; FVW2-NEXT: entry: ; FVW2-NEXT: br label [[VECTOR_BODY:%.*]] ; FVW2: vector.body: -; FVW2-NEXT: [[INDEX7:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; FVW2-NEXT: [[INDEX7:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE27:%.*]] ] ; FVW2-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[INDEX7]] ; FVW2-NEXT: [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <2 x i32>* ; FVW2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 4 @@ -128,33 +128,105 @@ ; FVW2-NEXT: [[TMP21:%.*]] = sext <2 x i32> [[WIDE_MASKED_LOAD11]] to <2 x i64> ; FVW2-NEXT: [[TMP22:%.*]] = sext <2 x i32> [[WIDE_MASKED_LOAD12]] to <2 x i64> ; FVW2-NEXT: [[TMP23:%.*]] = sext <2 x i32> [[WIDE_MASKED_LOAD13]] to <2 x i64> -; FVW2-NEXT: [[TMP24:%.*]] = getelementptr inbounds float, float* [[IN:%.*]], <2 x i64> [[TMP20]] -; FVW2-NEXT: [[TMP25:%.*]] = getelementptr inbounds float, float* [[IN]], <2 x i64> [[TMP21]] -; FVW2-NEXT: [[TMP26:%.*]] = getelementptr inbounds float, float* [[IN]], <2 x i64> [[TMP22]] -; FVW2-NEXT: [[TMP27:%.*]] = getelementptr inbounds float, float* [[IN]], <2 x i64> [[TMP23]] -; FVW2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP24]], i32 4, <2 x i1> [[TMP8]], <2 x float> undef) -; FVW2-NEXT: [[WIDE_MASKED_GATHER14:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP25]], i32 4, <2 x i1> [[TMP9]], <2 x float> undef) -; FVW2-NEXT: [[WIDE_MASKED_GATHER15:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP26]], i32 4, <2 x i1> [[TMP10]], <2 x float> undef) -; FVW2-NEXT: [[WIDE_MASKED_GATHER16:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP27]], i32 4, <2 x i1> [[TMP11]], <2 x float> undef) -; FVW2-NEXT: [[TMP28:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER]], -; FVW2-NEXT: [[TMP29:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER14]], -; FVW2-NEXT: [[TMP30:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER15]], -; FVW2-NEXT: [[TMP31:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER16]], -; FVW2-NEXT: [[TMP32:%.*]] = getelementptr float, float* [[OUT:%.*]], i64 [[INDEX7]] -; FVW2-NEXT: [[TMP33:%.*]] = bitcast float* [[TMP32]] to <2 x float>* -; FVW2-NEXT: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> [[TMP28]], <2 x float>* [[TMP33]], i32 4, <2 x i1> [[TMP8]]) -; FVW2-NEXT: [[TMP34:%.*]] = getelementptr float, float* [[TMP32]], i64 2 -; FVW2-NEXT: [[TMP35:%.*]] = bitcast float* [[TMP34]] to <2 x float>* -; FVW2-NEXT: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> [[TMP29]], <2 x float>* [[TMP35]], i32 4, <2 x i1> [[TMP9]]) -; FVW2-NEXT: [[TMP36:%.*]] = getelementptr float, float* [[TMP32]], i64 4 -; FVW2-NEXT: [[TMP37:%.*]] = bitcast float* [[TMP36]] to <2 x float>* -; FVW2-NEXT: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> [[TMP30]], <2 x float>* [[TMP37]], i32 4, <2 x i1> [[TMP10]]) -; FVW2-NEXT: [[TMP38:%.*]] = getelementptr float, float* [[TMP32]], i64 6 -; FVW2-NEXT: [[TMP39:%.*]] = bitcast float* [[TMP38]] to <2 x float>* -; FVW2-NEXT: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> [[TMP31]], <2 x float>* [[TMP39]], i32 4, <2 x i1> [[TMP11]]) +; FVW2-NEXT: [[TMP24:%.*]] = extractelement <2 x i1> [[TMP8]], i32 0 +; FVW2-NEXT: br i1 [[TMP24]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; FVW2: pred.load.if: +; FVW2-NEXT: [[TMP25:%.*]] = extractelement <2 x i64> [[TMP20]], i32 0 +; FVW2-NEXT: [[TMP26:%.*]] = getelementptr inbounds float, float* [[IN:%.*]], i64 [[TMP25]] +; FVW2-NEXT: [[TMP27:%.*]] = load float, float* [[TMP26]], align 4 +; FVW2-NEXT: [[TMP28:%.*]] = insertelement <2 x float> poison, float [[TMP27]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE]] +; FVW2: pred.load.continue: +; FVW2-NEXT: [[TMP29:%.*]] = phi <2 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP28]], [[PRED_LOAD_IF]] ] +; FVW2-NEXT: [[TMP30:%.*]] = extractelement <2 x i1> [[TMP8]], i32 1 +; FVW2-NEXT: br i1 [[TMP30]], label [[PRED_LOAD_IF14:%.*]], label [[PRED_LOAD_CONTINUE15:%.*]] +; FVW2: pred.load.if14: +; FVW2-NEXT: [[TMP31:%.*]] = extractelement <2 x i64> [[TMP20]], i32 1 +; FVW2-NEXT: [[TMP32:%.*]] = getelementptr inbounds float, float* [[IN]], i64 [[TMP31]] +; FVW2-NEXT: [[TMP33:%.*]] = load float, float* [[TMP32]], align 4 +; FVW2-NEXT: [[TMP34:%.*]] = insertelement <2 x float> [[TMP29]], float [[TMP33]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE15]] +; FVW2: pred.load.continue15: +; FVW2-NEXT: [[TMP35:%.*]] = phi <2 x float> [ [[TMP29]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP34]], [[PRED_LOAD_IF14]] ] +; FVW2-NEXT: [[TMP36:%.*]] = extractelement <2 x i1> [[TMP9]], i32 0 +; FVW2-NEXT: br i1 [[TMP36]], label [[PRED_LOAD_IF16:%.*]], label [[PRED_LOAD_CONTINUE17:%.*]] +; FVW2: pred.load.if16: +; FVW2-NEXT: [[TMP37:%.*]] = extractelement <2 x i64> [[TMP21]], i32 0 +; FVW2-NEXT: [[TMP38:%.*]] = getelementptr inbounds float, float* [[IN]], i64 [[TMP37]] +; FVW2-NEXT: [[TMP39:%.*]] = load float, float* [[TMP38]], align 4 +; FVW2-NEXT: [[TMP40:%.*]] = insertelement <2 x float> poison, float [[TMP39]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE17]] +; FVW2: pred.load.continue17: +; FVW2-NEXT: [[TMP41:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE15]] ], [ [[TMP40]], [[PRED_LOAD_IF16]] ] +; FVW2-NEXT: [[TMP42:%.*]] = extractelement <2 x i1> [[TMP9]], i32 1 +; FVW2-NEXT: br i1 [[TMP42]], label [[PRED_LOAD_IF18:%.*]], label [[PRED_LOAD_CONTINUE19:%.*]] +; FVW2: pred.load.if18: +; FVW2-NEXT: [[TMP43:%.*]] = extractelement <2 x i64> [[TMP21]], i32 1 +; FVW2-NEXT: [[TMP44:%.*]] = getelementptr inbounds float, float* [[IN]], i64 [[TMP43]] +; FVW2-NEXT: [[TMP45:%.*]] = load float, float* [[TMP44]], align 4 +; FVW2-NEXT: [[TMP46:%.*]] = insertelement <2 x float> [[TMP41]], float [[TMP45]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE19]] +; FVW2: pred.load.continue19: +; FVW2-NEXT: [[TMP47:%.*]] = phi <2 x float> [ [[TMP41]], [[PRED_LOAD_CONTINUE17]] ], [ [[TMP46]], [[PRED_LOAD_IF18]] ] +; FVW2-NEXT: [[TMP48:%.*]] = extractelement <2 x i1> [[TMP10]], i32 0 +; FVW2-NEXT: br i1 [[TMP48]], label [[PRED_LOAD_IF20:%.*]], label [[PRED_LOAD_CONTINUE21:%.*]] +; FVW2: pred.load.if20: +; FVW2-NEXT: [[TMP49:%.*]] = extractelement <2 x i64> [[TMP22]], i32 0 +; FVW2-NEXT: [[TMP50:%.*]] = getelementptr inbounds float, float* [[IN]], i64 [[TMP49]] +; FVW2-NEXT: [[TMP51:%.*]] = load float, float* [[TMP50]], align 4 +; FVW2-NEXT: [[TMP52:%.*]] = insertelement <2 x float> poison, float [[TMP51]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE21]] +; FVW2: pred.load.continue21: +; FVW2-NEXT: [[TMP53:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE19]] ], [ [[TMP52]], [[PRED_LOAD_IF20]] ] +; FVW2-NEXT: [[TMP54:%.*]] = extractelement <2 x i1> [[TMP10]], i32 1 +; FVW2-NEXT: br i1 [[TMP54]], label [[PRED_LOAD_IF22:%.*]], label [[PRED_LOAD_CONTINUE23:%.*]] +; FVW2: pred.load.if22: +; FVW2-NEXT: [[TMP55:%.*]] = extractelement <2 x i64> [[TMP22]], i32 1 +; FVW2-NEXT: [[TMP56:%.*]] = getelementptr inbounds float, float* [[IN]], i64 [[TMP55]] +; FVW2-NEXT: [[TMP57:%.*]] = load float, float* [[TMP56]], align 4 +; FVW2-NEXT: [[TMP58:%.*]] = insertelement <2 x float> [[TMP53]], float [[TMP57]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE23]] +; FVW2: pred.load.continue23: +; FVW2-NEXT: [[TMP59:%.*]] = phi <2 x float> [ [[TMP53]], [[PRED_LOAD_CONTINUE21]] ], [ [[TMP58]], [[PRED_LOAD_IF22]] ] +; FVW2-NEXT: [[TMP60:%.*]] = extractelement <2 x i1> [[TMP11]], i32 0 +; FVW2-NEXT: br i1 [[TMP60]], label [[PRED_LOAD_IF24:%.*]], label [[PRED_LOAD_CONTINUE25:%.*]] +; FVW2: pred.load.if24: +; FVW2-NEXT: [[TMP61:%.*]] = extractelement <2 x i64> [[TMP23]], i32 0 +; FVW2-NEXT: [[TMP62:%.*]] = getelementptr inbounds float, float* [[IN]], i64 [[TMP61]] +; FVW2-NEXT: [[TMP63:%.*]] = load float, float* [[TMP62]], align 4 +; FVW2-NEXT: [[TMP64:%.*]] = insertelement <2 x float> poison, float [[TMP63]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE25]] +; FVW2: pred.load.continue25: +; FVW2-NEXT: [[TMP65:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE23]] ], [ [[TMP64]], [[PRED_LOAD_IF24]] ] +; FVW2-NEXT: [[TMP66:%.*]] = extractelement <2 x i1> [[TMP11]], i32 1 +; FVW2-NEXT: br i1 [[TMP66]], label [[PRED_LOAD_IF26:%.*]], label [[PRED_LOAD_CONTINUE27]] +; FVW2: pred.load.if26: +; FVW2-NEXT: [[TMP67:%.*]] = extractelement <2 x i64> [[TMP23]], i32 1 +; FVW2-NEXT: [[TMP68:%.*]] = getelementptr inbounds float, float* [[IN]], i64 [[TMP67]] +; FVW2-NEXT: [[TMP69:%.*]] = load float, float* [[TMP68]], align 4 +; FVW2-NEXT: [[TMP70:%.*]] = insertelement <2 x float> [[TMP65]], float [[TMP69]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE27]] +; FVW2: pred.load.continue27: +; FVW2-NEXT: [[TMP71:%.*]] = phi <2 x float> [ [[TMP65]], [[PRED_LOAD_CONTINUE25]] ], [ [[TMP70]], [[PRED_LOAD_IF26]] ] +; FVW2-NEXT: [[TMP72:%.*]] = fadd <2 x float> [[TMP35]], +; FVW2-NEXT: [[TMP73:%.*]] = fadd <2 x float> [[TMP47]], +; FVW2-NEXT: [[TMP74:%.*]] = fadd <2 x float> [[TMP59]], +; FVW2-NEXT: [[TMP75:%.*]] = fadd <2 x float> [[TMP71]], +; FVW2-NEXT: [[TMP76:%.*]] = getelementptr float, float* [[OUT:%.*]], i64 [[INDEX7]] +; FVW2-NEXT: [[TMP77:%.*]] = bitcast float* [[TMP76]] to <2 x float>* +; FVW2-NEXT: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> [[TMP72]], <2 x float>* [[TMP77]], i32 4, <2 x i1> [[TMP8]]) +; FVW2-NEXT: [[TMP78:%.*]] = getelementptr float, float* [[TMP76]], i64 2 +; FVW2-NEXT: [[TMP79:%.*]] = bitcast float* [[TMP78]] to <2 x float>* +; FVW2-NEXT: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> [[TMP73]], <2 x float>* [[TMP79]], i32 4, <2 x i1> [[TMP9]]) +; FVW2-NEXT: [[TMP80:%.*]] = getelementptr float, float* [[TMP76]], i64 4 +; FVW2-NEXT: [[TMP81:%.*]] = bitcast float* [[TMP80]] to <2 x float>* +; FVW2-NEXT: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> [[TMP74]], <2 x float>* [[TMP81]], i32 4, <2 x i1> [[TMP10]]) +; FVW2-NEXT: [[TMP82:%.*]] = getelementptr float, float* [[TMP76]], i64 6 +; FVW2-NEXT: [[TMP83:%.*]] = bitcast float* [[TMP82]] to <2 x float>* +; FVW2-NEXT: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> [[TMP75]], <2 x float>* [[TMP83]], i32 4, <2 x i1> [[TMP11]]) ; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX7]], 8 -; FVW2-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096 -; FVW2-NEXT: br i1 [[TMP40]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; FVW2-NEXT: [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096 +; FVW2-NEXT: br i1 [[TMP84]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; FVW2: for.end: ; FVW2-NEXT: ret void ; @@ -365,40 +437,186 @@ ; FVW2-NEXT: entry: ; FVW2-NEXT: br label [[VECTOR_BODY:%.*]] ; FVW2: vector.body: -; FVW2-NEXT: [[INDEX7:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE9:%.*]] ] -; FVW2-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ , [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE9]] ] +; FVW2-NEXT: [[INDEX7:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE35:%.*]] ] ; FVW2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX7]], 4 ; FVW2-NEXT: [[TMP0:%.*]] = or i64 [[OFFSET_IDX]], 16 -; FVW2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[OFFSET_IDX]] -; FVW2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP0]] -; FVW2-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1]], align 4 -; FVW2-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP2]], align 4 -; FVW2-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i32 0 -; FVW2-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP4]], i32 1 -; FVW2-NEXT: [[TMP7:%.*]] = icmp sgt <2 x i32> [[TMP6]], zeroinitializer -; FVW2-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], %struct.In* [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1 -; FVW2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP8]], i32 4, <2 x i1> [[TMP7]], <2 x float> undef) -; FVW2-NEXT: [[TMP9:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER]], -; FVW2-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0 -; FVW2-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] +; FVW2-NEXT: [[TMP1:%.*]] = or i64 [[OFFSET_IDX]], 32 +; FVW2-NEXT: [[TMP2:%.*]] = or i64 [[OFFSET_IDX]], 48 +; FVW2-NEXT: [[TMP3:%.*]] = or i64 [[OFFSET_IDX]], 64 +; FVW2-NEXT: [[TMP4:%.*]] = or i64 [[OFFSET_IDX]], 80 +; FVW2-NEXT: [[TMP5:%.*]] = or i64 [[OFFSET_IDX]], 96 +; FVW2-NEXT: [[TMP6:%.*]] = or i64 [[OFFSET_IDX]], 112 +; FVW2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[OFFSET_IDX]] +; FVW2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP0]] +; FVW2-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP1]] +; FVW2-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP2]] +; FVW2-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP3]] +; FVW2-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP4]] +; FVW2-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP5]] +; FVW2-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP6]] +; FVW2-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP7]], align 4 +; FVW2-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP8]], align 4 +; FVW2-NEXT: [[TMP17:%.*]] = insertelement <2 x i32> poison, i32 [[TMP15]], i32 0 +; FVW2-NEXT: [[TMP18:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP16]], i32 1 +; FVW2-NEXT: [[TMP19:%.*]] = load i32, i32* [[TMP9]], align 4 +; FVW2-NEXT: [[TMP20:%.*]] = load i32, i32* [[TMP10]], align 4 +; FVW2-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> poison, i32 [[TMP19]], i32 0 +; FVW2-NEXT: [[TMP22:%.*]] = insertelement <2 x i32> [[TMP21]], i32 [[TMP20]], i32 1 +; FVW2-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP11]], align 4 +; FVW2-NEXT: [[TMP24:%.*]] = load i32, i32* [[TMP12]], align 4 +; FVW2-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> poison, i32 [[TMP23]], i32 0 +; FVW2-NEXT: [[TMP26:%.*]] = insertelement <2 x i32> [[TMP25]], i32 [[TMP24]], i32 1 +; FVW2-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP13]], align 4 +; FVW2-NEXT: [[TMP28:%.*]] = load i32, i32* [[TMP14]], align 4 +; FVW2-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> poison, i32 [[TMP27]], i32 0 +; FVW2-NEXT: [[TMP30:%.*]] = insertelement <2 x i32> [[TMP29]], i32 [[TMP28]], i32 1 +; FVW2-NEXT: [[TMP31:%.*]] = icmp sgt <2 x i32> [[TMP18]], zeroinitializer +; FVW2-NEXT: [[TMP32:%.*]] = icmp sgt <2 x i32> [[TMP22]], zeroinitializer +; FVW2-NEXT: [[TMP33:%.*]] = icmp sgt <2 x i32> [[TMP26]], zeroinitializer +; FVW2-NEXT: [[TMP34:%.*]] = icmp sgt <2 x i32> [[TMP30]], zeroinitializer +; FVW2-NEXT: [[TMP35:%.*]] = extractelement <2 x i1> [[TMP31]], i32 0 +; FVW2-NEXT: br i1 [[TMP35]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; FVW2: pred.load.if: +; FVW2-NEXT: [[TMP36:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], %struct.In* [[IN:%.*]], i64 [[OFFSET_IDX]], i32 1 +; FVW2-NEXT: [[TMP37:%.*]] = load float, float* [[TMP36]], align 4 +; FVW2-NEXT: [[TMP38:%.*]] = insertelement <2 x float> poison, float [[TMP37]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE]] +; FVW2: pred.load.continue: +; FVW2-NEXT: [[TMP39:%.*]] = phi <2 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP38]], [[PRED_LOAD_IF]] ] +; FVW2-NEXT: [[TMP40:%.*]] = extractelement <2 x i1> [[TMP31]], i32 1 +; FVW2-NEXT: br i1 [[TMP40]], label [[PRED_LOAD_IF8:%.*]], label [[PRED_LOAD_CONTINUE9:%.*]] +; FVW2: pred.load.if8: +; FVW2-NEXT: [[TMP41:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP0]], i32 1 +; FVW2-NEXT: [[TMP42:%.*]] = load float, float* [[TMP41]], align 4 +; FVW2-NEXT: [[TMP43:%.*]] = insertelement <2 x float> [[TMP39]], float [[TMP42]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE9]] +; FVW2: pred.load.continue9: +; FVW2-NEXT: [[TMP44:%.*]] = phi <2 x float> [ [[TMP39]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP43]], [[PRED_LOAD_IF8]] ] +; FVW2-NEXT: [[TMP45:%.*]] = extractelement <2 x i1> [[TMP32]], i32 0 +; FVW2-NEXT: br i1 [[TMP45]], label [[PRED_LOAD_IF10:%.*]], label [[PRED_LOAD_CONTINUE11:%.*]] +; FVW2: pred.load.if10: +; FVW2-NEXT: [[TMP46:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP1]], i32 1 +; FVW2-NEXT: [[TMP47:%.*]] = load float, float* [[TMP46]], align 4 +; FVW2-NEXT: [[TMP48:%.*]] = insertelement <2 x float> poison, float [[TMP47]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE11]] +; FVW2: pred.load.continue11: +; FVW2-NEXT: [[TMP49:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE9]] ], [ [[TMP48]], [[PRED_LOAD_IF10]] ] +; FVW2-NEXT: [[TMP50:%.*]] = extractelement <2 x i1> [[TMP32]], i32 1 +; FVW2-NEXT: br i1 [[TMP50]], label [[PRED_LOAD_IF12:%.*]], label [[PRED_LOAD_CONTINUE13:%.*]] +; FVW2: pred.load.if12: +; FVW2-NEXT: [[TMP51:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP2]], i32 1 +; FVW2-NEXT: [[TMP52:%.*]] = load float, float* [[TMP51]], align 4 +; FVW2-NEXT: [[TMP53:%.*]] = insertelement <2 x float> [[TMP49]], float [[TMP52]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE13]] +; FVW2: pred.load.continue13: +; FVW2-NEXT: [[TMP54:%.*]] = phi <2 x float> [ [[TMP49]], [[PRED_LOAD_CONTINUE11]] ], [ [[TMP53]], [[PRED_LOAD_IF12]] ] +; FVW2-NEXT: [[TMP55:%.*]] = extractelement <2 x i1> [[TMP33]], i32 0 +; FVW2-NEXT: br i1 [[TMP55]], label [[PRED_LOAD_IF14:%.*]], label [[PRED_LOAD_CONTINUE15:%.*]] +; FVW2: pred.load.if14: +; FVW2-NEXT: [[TMP56:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP3]], i32 1 +; FVW2-NEXT: [[TMP57:%.*]] = load float, float* [[TMP56]], align 4 +; FVW2-NEXT: [[TMP58:%.*]] = insertelement <2 x float> poison, float [[TMP57]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE15]] +; FVW2: pred.load.continue15: +; FVW2-NEXT: [[TMP59:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE13]] ], [ [[TMP58]], [[PRED_LOAD_IF14]] ] +; FVW2-NEXT: [[TMP60:%.*]] = extractelement <2 x i1> [[TMP33]], i32 1 +; FVW2-NEXT: br i1 [[TMP60]], label [[PRED_LOAD_IF16:%.*]], label [[PRED_LOAD_CONTINUE17:%.*]] +; FVW2: pred.load.if16: +; FVW2-NEXT: [[TMP61:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP4]], i32 1 +; FVW2-NEXT: [[TMP62:%.*]] = load float, float* [[TMP61]], align 4 +; FVW2-NEXT: [[TMP63:%.*]] = insertelement <2 x float> [[TMP59]], float [[TMP62]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE17]] +; FVW2: pred.load.continue17: +; FVW2-NEXT: [[TMP64:%.*]] = phi <2 x float> [ [[TMP59]], [[PRED_LOAD_CONTINUE15]] ], [ [[TMP63]], [[PRED_LOAD_IF16]] ] +; FVW2-NEXT: [[TMP65:%.*]] = extractelement <2 x i1> [[TMP34]], i32 0 +; FVW2-NEXT: br i1 [[TMP65]], label [[PRED_LOAD_IF18:%.*]], label [[PRED_LOAD_CONTINUE19:%.*]] +; FVW2: pred.load.if18: +; FVW2-NEXT: [[TMP66:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP5]], i32 1 +; FVW2-NEXT: [[TMP67:%.*]] = load float, float* [[TMP66]], align 4 +; FVW2-NEXT: [[TMP68:%.*]] = insertelement <2 x float> poison, float [[TMP67]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE19]] +; FVW2: pred.load.continue19: +; FVW2-NEXT: [[TMP69:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE17]] ], [ [[TMP68]], [[PRED_LOAD_IF18]] ] +; FVW2-NEXT: [[TMP70:%.*]] = extractelement <2 x i1> [[TMP34]], i32 1 +; FVW2-NEXT: br i1 [[TMP70]], label [[PRED_LOAD_IF20:%.*]], label [[PRED_LOAD_CONTINUE21:%.*]] +; FVW2: pred.load.if20: +; FVW2-NEXT: [[TMP71:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP6]], i32 1 +; FVW2-NEXT: [[TMP72:%.*]] = load float, float* [[TMP71]], align 4 +; FVW2-NEXT: [[TMP73:%.*]] = insertelement <2 x float> [[TMP69]], float [[TMP72]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE21]] +; FVW2: pred.load.continue21: +; FVW2-NEXT: [[TMP74:%.*]] = phi <2 x float> [ [[TMP69]], [[PRED_LOAD_CONTINUE19]] ], [ [[TMP73]], [[PRED_LOAD_IF20]] ] +; FVW2-NEXT: [[TMP75:%.*]] = fadd <2 x float> [[TMP44]], +; FVW2-NEXT: [[TMP76:%.*]] = fadd <2 x float> [[TMP54]], +; FVW2-NEXT: [[TMP77:%.*]] = fadd <2 x float> [[TMP64]], +; FVW2-NEXT: [[TMP78:%.*]] = fadd <2 x float> [[TMP74]], +; FVW2-NEXT: [[TMP79:%.*]] = extractelement <2 x i1> [[TMP31]], i32 0 +; FVW2-NEXT: br i1 [[TMP79]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] ; FVW2: pred.store.if: -; FVW2-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, float* [[OUT:%.*]], i64 [[OFFSET_IDX]] -; FVW2-NEXT: [[TMP12:%.*]] = extractelement <2 x float> [[TMP9]], i32 0 -; FVW2-NEXT: store float [[TMP12]], float* [[TMP11]], align 4 +; FVW2-NEXT: [[TMP80:%.*]] = getelementptr inbounds float, float* [[OUT:%.*]], i64 [[OFFSET_IDX]] +; FVW2-NEXT: [[TMP81:%.*]] = extractelement <2 x float> [[TMP75]], i32 0 +; FVW2-NEXT: store float [[TMP81]], float* [[TMP80]], align 4 ; FVW2-NEXT: br label [[PRED_STORE_CONTINUE]] ; FVW2: pred.store.continue: -; FVW2-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1 -; FVW2-NEXT: br i1 [[TMP13]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9]] -; FVW2: pred.store.if8: -; FVW2-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP0]] -; FVW2-NEXT: [[TMP15:%.*]] = extractelement <2 x float> [[TMP9]], i32 1 -; FVW2-NEXT: store float [[TMP15]], float* [[TMP14]], align 4 -; FVW2-NEXT: br label [[PRED_STORE_CONTINUE9]] -; FVW2: pred.store.continue9: -; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX7]], 2 -; FVW2-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], -; FVW2-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 -; FVW2-NEXT: br i1 [[TMP16]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]] +; FVW2-NEXT: [[TMP82:%.*]] = extractelement <2 x i1> [[TMP31]], i32 1 +; FVW2-NEXT: br i1 [[TMP82]], label [[PRED_STORE_IF22:%.*]], label [[PRED_STORE_CONTINUE23:%.*]] +; FVW2: pred.store.if22: +; FVW2-NEXT: [[TMP83:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP0]] +; FVW2-NEXT: [[TMP84:%.*]] = extractelement <2 x float> [[TMP75]], i32 1 +; FVW2-NEXT: store float [[TMP84]], float* [[TMP83]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE23]] +; FVW2: pred.store.continue23: +; FVW2-NEXT: [[TMP85:%.*]] = extractelement <2 x i1> [[TMP32]], i32 0 +; FVW2-NEXT: br i1 [[TMP85]], label [[PRED_STORE_IF24:%.*]], label [[PRED_STORE_CONTINUE25:%.*]] +; FVW2: pred.store.if24: +; FVW2-NEXT: [[TMP86:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP1]] +; FVW2-NEXT: [[TMP87:%.*]] = extractelement <2 x float> [[TMP76]], i32 0 +; FVW2-NEXT: store float [[TMP87]], float* [[TMP86]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE25]] +; FVW2: pred.store.continue25: +; FVW2-NEXT: [[TMP88:%.*]] = extractelement <2 x i1> [[TMP32]], i32 1 +; FVW2-NEXT: br i1 [[TMP88]], label [[PRED_STORE_IF26:%.*]], label [[PRED_STORE_CONTINUE27:%.*]] +; FVW2: pred.store.if26: +; FVW2-NEXT: [[TMP89:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP2]] +; FVW2-NEXT: [[TMP90:%.*]] = extractelement <2 x float> [[TMP76]], i32 1 +; FVW2-NEXT: store float [[TMP90]], float* [[TMP89]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE27]] +; FVW2: pred.store.continue27: +; FVW2-NEXT: [[TMP91:%.*]] = extractelement <2 x i1> [[TMP33]], i32 0 +; FVW2-NEXT: br i1 [[TMP91]], label [[PRED_STORE_IF28:%.*]], label [[PRED_STORE_CONTINUE29:%.*]] +; FVW2: pred.store.if28: +; FVW2-NEXT: [[TMP92:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP3]] +; FVW2-NEXT: [[TMP93:%.*]] = extractelement <2 x float> [[TMP77]], i32 0 +; FVW2-NEXT: store float [[TMP93]], float* [[TMP92]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE29]] +; FVW2: pred.store.continue29: +; FVW2-NEXT: [[TMP94:%.*]] = extractelement <2 x i1> [[TMP33]], i32 1 +; FVW2-NEXT: br i1 [[TMP94]], label [[PRED_STORE_IF30:%.*]], label [[PRED_STORE_CONTINUE31:%.*]] +; FVW2: pred.store.if30: +; FVW2-NEXT: [[TMP95:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP4]] +; FVW2-NEXT: [[TMP96:%.*]] = extractelement <2 x float> [[TMP77]], i32 1 +; FVW2-NEXT: store float [[TMP96]], float* [[TMP95]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE31]] +; FVW2: pred.store.continue31: +; FVW2-NEXT: [[TMP97:%.*]] = extractelement <2 x i1> [[TMP34]], i32 0 +; FVW2-NEXT: br i1 [[TMP97]], label [[PRED_STORE_IF32:%.*]], label [[PRED_STORE_CONTINUE33:%.*]] +; FVW2: pred.store.if32: +; FVW2-NEXT: [[TMP98:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP5]] +; FVW2-NEXT: [[TMP99:%.*]] = extractelement <2 x float> [[TMP78]], i32 0 +; FVW2-NEXT: store float [[TMP99]], float* [[TMP98]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE33]] +; FVW2: pred.store.continue33: +; FVW2-NEXT: [[TMP100:%.*]] = extractelement <2 x i1> [[TMP34]], i32 1 +; FVW2-NEXT: br i1 [[TMP100]], label [[PRED_STORE_IF34:%.*]], label [[PRED_STORE_CONTINUE35]] +; FVW2: pred.store.if34: +; FVW2-NEXT: [[TMP101:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP6]] +; FVW2-NEXT: [[TMP102:%.*]] = extractelement <2 x float> [[TMP78]], i32 1 +; FVW2-NEXT: store float [[TMP102]], float* [[TMP101]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE35]] +; FVW2: pred.store.continue35: +; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX7]], 8 +; FVW2-NEXT: [[TMP103:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 +; FVW2-NEXT: br i1 [[TMP103]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]] ; FVW2: for.end: ; FVW2-NEXT: ret void ; @@ -610,40 +828,186 @@ ; FVW2-NEXT: entry: ; FVW2-NEXT: br label [[VECTOR_BODY:%.*]] ; FVW2: vector.body: -; FVW2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE8:%.*]] ] -; FVW2-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ , [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE8]] ] +; FVW2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE34:%.*]] ] ; FVW2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 4 ; FVW2-NEXT: [[TMP0:%.*]] = or i64 [[OFFSET_IDX]], 16 -; FVW2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[OFFSET_IDX]] -; FVW2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP0]] -; FVW2-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1]], align 4 -; FVW2-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP2]], align 4 -; FVW2-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i32 0 -; FVW2-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP4]], i32 1 -; FVW2-NEXT: [[TMP7:%.*]] = icmp sgt <2 x i32> [[TMP6]], zeroinitializer -; FVW2-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], %struct.In* [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1 -; FVW2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP8]], i32 4, <2 x i1> [[TMP7]], <2 x float> undef) -; FVW2-NEXT: [[TMP9:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER]], -; FVW2-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0 -; FVW2-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] +; FVW2-NEXT: [[TMP1:%.*]] = or i64 [[OFFSET_IDX]], 32 +; FVW2-NEXT: [[TMP2:%.*]] = or i64 [[OFFSET_IDX]], 48 +; FVW2-NEXT: [[TMP3:%.*]] = or i64 [[OFFSET_IDX]], 64 +; FVW2-NEXT: [[TMP4:%.*]] = or i64 [[OFFSET_IDX]], 80 +; FVW2-NEXT: [[TMP5:%.*]] = or i64 [[OFFSET_IDX]], 96 +; FVW2-NEXT: [[TMP6:%.*]] = or i64 [[OFFSET_IDX]], 112 +; FVW2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[OFFSET_IDX]] +; FVW2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP0]] +; FVW2-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP1]] +; FVW2-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP2]] +; FVW2-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP3]] +; FVW2-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP4]] +; FVW2-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP5]] +; FVW2-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP6]] +; FVW2-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP7]], align 4 +; FVW2-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP8]], align 4 +; FVW2-NEXT: [[TMP17:%.*]] = insertelement <2 x i32> poison, i32 [[TMP15]], i32 0 +; FVW2-NEXT: [[TMP18:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP16]], i32 1 +; FVW2-NEXT: [[TMP19:%.*]] = load i32, i32* [[TMP9]], align 4 +; FVW2-NEXT: [[TMP20:%.*]] = load i32, i32* [[TMP10]], align 4 +; FVW2-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> poison, i32 [[TMP19]], i32 0 +; FVW2-NEXT: [[TMP22:%.*]] = insertelement <2 x i32> [[TMP21]], i32 [[TMP20]], i32 1 +; FVW2-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP11]], align 4 +; FVW2-NEXT: [[TMP24:%.*]] = load i32, i32* [[TMP12]], align 4 +; FVW2-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> poison, i32 [[TMP23]], i32 0 +; FVW2-NEXT: [[TMP26:%.*]] = insertelement <2 x i32> [[TMP25]], i32 [[TMP24]], i32 1 +; FVW2-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP13]], align 4 +; FVW2-NEXT: [[TMP28:%.*]] = load i32, i32* [[TMP14]], align 4 +; FVW2-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> poison, i32 [[TMP27]], i32 0 +; FVW2-NEXT: [[TMP30:%.*]] = insertelement <2 x i32> [[TMP29]], i32 [[TMP28]], i32 1 +; FVW2-NEXT: [[TMP31:%.*]] = icmp sgt <2 x i32> [[TMP18]], zeroinitializer +; FVW2-NEXT: [[TMP32:%.*]] = icmp sgt <2 x i32> [[TMP22]], zeroinitializer +; FVW2-NEXT: [[TMP33:%.*]] = icmp sgt <2 x i32> [[TMP26]], zeroinitializer +; FVW2-NEXT: [[TMP34:%.*]] = icmp sgt <2 x i32> [[TMP30]], zeroinitializer +; FVW2-NEXT: [[TMP35:%.*]] = extractelement <2 x i1> [[TMP31]], i32 0 +; FVW2-NEXT: br i1 [[TMP35]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; FVW2: pred.load.if: +; FVW2-NEXT: [[TMP36:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], %struct.In* [[IN:%.*]], i64 [[OFFSET_IDX]], i32 1 +; FVW2-NEXT: [[TMP37:%.*]] = load float, float* [[TMP36]], align 4 +; FVW2-NEXT: [[TMP38:%.*]] = insertelement <2 x float> poison, float [[TMP37]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE]] +; FVW2: pred.load.continue: +; FVW2-NEXT: [[TMP39:%.*]] = phi <2 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP38]], [[PRED_LOAD_IF]] ] +; FVW2-NEXT: [[TMP40:%.*]] = extractelement <2 x i1> [[TMP31]], i32 1 +; FVW2-NEXT: br i1 [[TMP40]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]] +; FVW2: pred.load.if7: +; FVW2-NEXT: [[TMP41:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP0]], i32 1 +; FVW2-NEXT: [[TMP42:%.*]] = load float, float* [[TMP41]], align 4 +; FVW2-NEXT: [[TMP43:%.*]] = insertelement <2 x float> [[TMP39]], float [[TMP42]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE8]] +; FVW2: pred.load.continue8: +; FVW2-NEXT: [[TMP44:%.*]] = phi <2 x float> [ [[TMP39]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP43]], [[PRED_LOAD_IF7]] ] +; FVW2-NEXT: [[TMP45:%.*]] = extractelement <2 x i1> [[TMP32]], i32 0 +; FVW2-NEXT: br i1 [[TMP45]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]] +; FVW2: pred.load.if9: +; FVW2-NEXT: [[TMP46:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP1]], i32 1 +; FVW2-NEXT: [[TMP47:%.*]] = load float, float* [[TMP46]], align 4 +; FVW2-NEXT: [[TMP48:%.*]] = insertelement <2 x float> poison, float [[TMP47]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE10]] +; FVW2: pred.load.continue10: +; FVW2-NEXT: [[TMP49:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE8]] ], [ [[TMP48]], [[PRED_LOAD_IF9]] ] +; FVW2-NEXT: [[TMP50:%.*]] = extractelement <2 x i1> [[TMP32]], i32 1 +; FVW2-NEXT: br i1 [[TMP50]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]] +; FVW2: pred.load.if11: +; FVW2-NEXT: [[TMP51:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP2]], i32 1 +; FVW2-NEXT: [[TMP52:%.*]] = load float, float* [[TMP51]], align 4 +; FVW2-NEXT: [[TMP53:%.*]] = insertelement <2 x float> [[TMP49]], float [[TMP52]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE12]] +; FVW2: pred.load.continue12: +; FVW2-NEXT: [[TMP54:%.*]] = phi <2 x float> [ [[TMP49]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP53]], [[PRED_LOAD_IF11]] ] +; FVW2-NEXT: [[TMP55:%.*]] = extractelement <2 x i1> [[TMP33]], i32 0 +; FVW2-NEXT: br i1 [[TMP55]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]] +; FVW2: pred.load.if13: +; FVW2-NEXT: [[TMP56:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP3]], i32 1 +; FVW2-NEXT: [[TMP57:%.*]] = load float, float* [[TMP56]], align 4 +; FVW2-NEXT: [[TMP58:%.*]] = insertelement <2 x float> poison, float [[TMP57]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE14]] +; FVW2: pred.load.continue14: +; FVW2-NEXT: [[TMP59:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE12]] ], [ [[TMP58]], [[PRED_LOAD_IF13]] ] +; FVW2-NEXT: [[TMP60:%.*]] = extractelement <2 x i1> [[TMP33]], i32 1 +; FVW2-NEXT: br i1 [[TMP60]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]] +; FVW2: pred.load.if15: +; FVW2-NEXT: [[TMP61:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP4]], i32 1 +; FVW2-NEXT: [[TMP62:%.*]] = load float, float* [[TMP61]], align 4 +; FVW2-NEXT: [[TMP63:%.*]] = insertelement <2 x float> [[TMP59]], float [[TMP62]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE16]] +; FVW2: pred.load.continue16: +; FVW2-NEXT: [[TMP64:%.*]] = phi <2 x float> [ [[TMP59]], [[PRED_LOAD_CONTINUE14]] ], [ [[TMP63]], [[PRED_LOAD_IF15]] ] +; FVW2-NEXT: [[TMP65:%.*]] = extractelement <2 x i1> [[TMP34]], i32 0 +; FVW2-NEXT: br i1 [[TMP65]], label [[PRED_LOAD_IF17:%.*]], label [[PRED_LOAD_CONTINUE18:%.*]] +; FVW2: pred.load.if17: +; FVW2-NEXT: [[TMP66:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP5]], i32 1 +; FVW2-NEXT: [[TMP67:%.*]] = load float, float* [[TMP66]], align 4 +; FVW2-NEXT: [[TMP68:%.*]] = insertelement <2 x float> poison, float [[TMP67]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE18]] +; FVW2: pred.load.continue18: +; FVW2-NEXT: [[TMP69:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE16]] ], [ [[TMP68]], [[PRED_LOAD_IF17]] ] +; FVW2-NEXT: [[TMP70:%.*]] = extractelement <2 x i1> [[TMP34]], i32 1 +; FVW2-NEXT: br i1 [[TMP70]], label [[PRED_LOAD_IF19:%.*]], label [[PRED_LOAD_CONTINUE20:%.*]] +; FVW2: pred.load.if19: +; FVW2-NEXT: [[TMP71:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP6]], i32 1 +; FVW2-NEXT: [[TMP72:%.*]] = load float, float* [[TMP71]], align 4 +; FVW2-NEXT: [[TMP73:%.*]] = insertelement <2 x float> [[TMP69]], float [[TMP72]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE20]] +; FVW2: pred.load.continue20: +; FVW2-NEXT: [[TMP74:%.*]] = phi <2 x float> [ [[TMP69]], [[PRED_LOAD_CONTINUE18]] ], [ [[TMP73]], [[PRED_LOAD_IF19]] ] +; FVW2-NEXT: [[TMP75:%.*]] = fadd <2 x float> [[TMP44]], +; FVW2-NEXT: [[TMP76:%.*]] = fadd <2 x float> [[TMP54]], +; FVW2-NEXT: [[TMP77:%.*]] = fadd <2 x float> [[TMP64]], +; FVW2-NEXT: [[TMP78:%.*]] = fadd <2 x float> [[TMP74]], +; FVW2-NEXT: [[TMP79:%.*]] = extractelement <2 x i1> [[TMP31]], i32 0 +; FVW2-NEXT: br i1 [[TMP79]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] ; FVW2: pred.store.if: -; FVW2-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_OUT:%.*]], %struct.Out* [[OUT:%.*]], i64 [[OFFSET_IDX]], i32 1 -; FVW2-NEXT: [[TMP12:%.*]] = extractelement <2 x float> [[TMP9]], i32 0 -; FVW2-NEXT: store float [[TMP12]], float* [[TMP11]], align 4 +; FVW2-NEXT: [[TMP80:%.*]] = getelementptr inbounds [[STRUCT_OUT:%.*]], %struct.Out* [[OUT:%.*]], i64 [[OFFSET_IDX]], i32 1 +; FVW2-NEXT: [[TMP81:%.*]] = extractelement <2 x float> [[TMP75]], i32 0 +; FVW2-NEXT: store float [[TMP81]], float* [[TMP80]], align 4 ; FVW2-NEXT: br label [[PRED_STORE_CONTINUE]] ; FVW2: pred.store.continue: -; FVW2-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1 -; FVW2-NEXT: br i1 [[TMP13]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8]] -; FVW2: pred.store.if7: -; FVW2-NEXT: [[TMP14:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP0]], i32 1 -; FVW2-NEXT: [[TMP15:%.*]] = extractelement <2 x float> [[TMP9]], i32 1 -; FVW2-NEXT: store float [[TMP15]], float* [[TMP14]], align 4 -; FVW2-NEXT: br label [[PRED_STORE_CONTINUE8]] -; FVW2: pred.store.continue8: -; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 -; FVW2-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], -; FVW2-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 -; FVW2-NEXT: br i1 [[TMP16]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; FVW2-NEXT: [[TMP82:%.*]] = extractelement <2 x i1> [[TMP31]], i32 1 +; FVW2-NEXT: br i1 [[TMP82]], label [[PRED_STORE_IF21:%.*]], label [[PRED_STORE_CONTINUE22:%.*]] +; FVW2: pred.store.if21: +; FVW2-NEXT: [[TMP83:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP0]], i32 1 +; FVW2-NEXT: [[TMP84:%.*]] = extractelement <2 x float> [[TMP75]], i32 1 +; FVW2-NEXT: store float [[TMP84]], float* [[TMP83]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE22]] +; FVW2: pred.store.continue22: +; FVW2-NEXT: [[TMP85:%.*]] = extractelement <2 x i1> [[TMP32]], i32 0 +; FVW2-NEXT: br i1 [[TMP85]], label [[PRED_STORE_IF23:%.*]], label [[PRED_STORE_CONTINUE24:%.*]] +; FVW2: pred.store.if23: +; FVW2-NEXT: [[TMP86:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP1]], i32 1 +; FVW2-NEXT: [[TMP87:%.*]] = extractelement <2 x float> [[TMP76]], i32 0 +; FVW2-NEXT: store float [[TMP87]], float* [[TMP86]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE24]] +; FVW2: pred.store.continue24: +; FVW2-NEXT: [[TMP88:%.*]] = extractelement <2 x i1> [[TMP32]], i32 1 +; FVW2-NEXT: br i1 [[TMP88]], label [[PRED_STORE_IF25:%.*]], label [[PRED_STORE_CONTINUE26:%.*]] +; FVW2: pred.store.if25: +; FVW2-NEXT: [[TMP89:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP2]], i32 1 +; FVW2-NEXT: [[TMP90:%.*]] = extractelement <2 x float> [[TMP76]], i32 1 +; FVW2-NEXT: store float [[TMP90]], float* [[TMP89]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE26]] +; FVW2: pred.store.continue26: +; FVW2-NEXT: [[TMP91:%.*]] = extractelement <2 x i1> [[TMP33]], i32 0 +; FVW2-NEXT: br i1 [[TMP91]], label [[PRED_STORE_IF27:%.*]], label [[PRED_STORE_CONTINUE28:%.*]] +; FVW2: pred.store.if27: +; FVW2-NEXT: [[TMP92:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP3]], i32 1 +; FVW2-NEXT: [[TMP93:%.*]] = extractelement <2 x float> [[TMP77]], i32 0 +; FVW2-NEXT: store float [[TMP93]], float* [[TMP92]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE28]] +; FVW2: pred.store.continue28: +; FVW2-NEXT: [[TMP94:%.*]] = extractelement <2 x i1> [[TMP33]], i32 1 +; FVW2-NEXT: br i1 [[TMP94]], label [[PRED_STORE_IF29:%.*]], label [[PRED_STORE_CONTINUE30:%.*]] +; FVW2: pred.store.if29: +; FVW2-NEXT: [[TMP95:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP4]], i32 1 +; FVW2-NEXT: [[TMP96:%.*]] = extractelement <2 x float> [[TMP77]], i32 1 +; FVW2-NEXT: store float [[TMP96]], float* [[TMP95]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE30]] +; FVW2: pred.store.continue30: +; FVW2-NEXT: [[TMP97:%.*]] = extractelement <2 x i1> [[TMP34]], i32 0 +; FVW2-NEXT: br i1 [[TMP97]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE32:%.*]] +; FVW2: pred.store.if31: +; FVW2-NEXT: [[TMP98:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP5]], i32 1 +; FVW2-NEXT: [[TMP99:%.*]] = extractelement <2 x float> [[TMP78]], i32 0 +; FVW2-NEXT: store float [[TMP99]], float* [[TMP98]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE32]] +; FVW2: pred.store.continue32: +; FVW2-NEXT: [[TMP100:%.*]] = extractelement <2 x i1> [[TMP34]], i32 1 +; FVW2-NEXT: br i1 [[TMP100]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34]] +; FVW2: pred.store.if33: +; FVW2-NEXT: [[TMP101:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP6]], i32 1 +; FVW2-NEXT: [[TMP102:%.*]] = extractelement <2 x float> [[TMP78]], i32 1 +; FVW2-NEXT: store float [[TMP102]], float* [[TMP101]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE34]] +; FVW2: pred.store.continue34: +; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 +; FVW2-NEXT: [[TMP103:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 +; FVW2-NEXT: br i1 [[TMP103]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; FVW2: for.end: ; FVW2-NEXT: ret void ; @@ -841,40 +1205,186 @@ ; FVW2-NEXT: entry: ; FVW2-NEXT: br label [[VECTOR_BODY:%.*]] ; FVW2: vector.body: -; FVW2-NEXT: [[INDEX7:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE9:%.*]] ] -; FVW2-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ , [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE9]] ] +; FVW2-NEXT: [[INDEX7:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE35:%.*]] ] ; FVW2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX7]], 4 ; FVW2-NEXT: [[TMP0:%.*]] = or i64 [[OFFSET_IDX]], 16 -; FVW2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[OFFSET_IDX]] -; FVW2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP0]] -; FVW2-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1]], align 4 -; FVW2-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP2]], align 4 -; FVW2-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i32 0 -; FVW2-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP4]], i32 1 -; FVW2-NEXT: [[TMP7:%.*]] = icmp sgt <2 x i32> [[TMP6]], zeroinitializer -; FVW2-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], [[STRUCT_IN]] addrspace(1)* [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1 -; FVW2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p1f32(<2 x float addrspace(1)*> [[TMP8]], i32 4, <2 x i1> [[TMP7]], <2 x float> undef) -; FVW2-NEXT: [[TMP9:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER]], -; FVW2-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0 -; FVW2-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] +; FVW2-NEXT: [[TMP1:%.*]] = or i64 [[OFFSET_IDX]], 32 +; FVW2-NEXT: [[TMP2:%.*]] = or i64 [[OFFSET_IDX]], 48 +; FVW2-NEXT: [[TMP3:%.*]] = or i64 [[OFFSET_IDX]], 64 +; FVW2-NEXT: [[TMP4:%.*]] = or i64 [[OFFSET_IDX]], 80 +; FVW2-NEXT: [[TMP5:%.*]] = or i64 [[OFFSET_IDX]], 96 +; FVW2-NEXT: [[TMP6:%.*]] = or i64 [[OFFSET_IDX]], 112 +; FVW2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[OFFSET_IDX]] +; FVW2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP0]] +; FVW2-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP1]] +; FVW2-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP2]] +; FVW2-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP3]] +; FVW2-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP4]] +; FVW2-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP5]] +; FVW2-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP6]] +; FVW2-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP7]], align 4 +; FVW2-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP8]], align 4 +; FVW2-NEXT: [[TMP17:%.*]] = insertelement <2 x i32> poison, i32 [[TMP15]], i32 0 +; FVW2-NEXT: [[TMP18:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP16]], i32 1 +; FVW2-NEXT: [[TMP19:%.*]] = load i32, i32* [[TMP9]], align 4 +; FVW2-NEXT: [[TMP20:%.*]] = load i32, i32* [[TMP10]], align 4 +; FVW2-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> poison, i32 [[TMP19]], i32 0 +; FVW2-NEXT: [[TMP22:%.*]] = insertelement <2 x i32> [[TMP21]], i32 [[TMP20]], i32 1 +; FVW2-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP11]], align 4 +; FVW2-NEXT: [[TMP24:%.*]] = load i32, i32* [[TMP12]], align 4 +; FVW2-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> poison, i32 [[TMP23]], i32 0 +; FVW2-NEXT: [[TMP26:%.*]] = insertelement <2 x i32> [[TMP25]], i32 [[TMP24]], i32 1 +; FVW2-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP13]], align 4 +; FVW2-NEXT: [[TMP28:%.*]] = load i32, i32* [[TMP14]], align 4 +; FVW2-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> poison, i32 [[TMP27]], i32 0 +; FVW2-NEXT: [[TMP30:%.*]] = insertelement <2 x i32> [[TMP29]], i32 [[TMP28]], i32 1 +; FVW2-NEXT: [[TMP31:%.*]] = icmp sgt <2 x i32> [[TMP18]], zeroinitializer +; FVW2-NEXT: [[TMP32:%.*]] = icmp sgt <2 x i32> [[TMP22]], zeroinitializer +; FVW2-NEXT: [[TMP33:%.*]] = icmp sgt <2 x i32> [[TMP26]], zeroinitializer +; FVW2-NEXT: [[TMP34:%.*]] = icmp sgt <2 x i32> [[TMP30]], zeroinitializer +; FVW2-NEXT: [[TMP35:%.*]] = extractelement <2 x i1> [[TMP31]], i32 0 +; FVW2-NEXT: br i1 [[TMP35]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; FVW2: pred.load.if: +; FVW2-NEXT: [[TMP36:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], [[STRUCT_IN]] addrspace(1)* [[IN:%.*]], i64 [[OFFSET_IDX]], i32 1 +; FVW2-NEXT: [[TMP37:%.*]] = load float, float addrspace(1)* [[TMP36]], align 4 +; FVW2-NEXT: [[TMP38:%.*]] = insertelement <2 x float> poison, float [[TMP37]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE]] +; FVW2: pred.load.continue: +; FVW2-NEXT: [[TMP39:%.*]] = phi <2 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP38]], [[PRED_LOAD_IF]] ] +; FVW2-NEXT: [[TMP40:%.*]] = extractelement <2 x i1> [[TMP31]], i32 1 +; FVW2-NEXT: br i1 [[TMP40]], label [[PRED_LOAD_IF8:%.*]], label [[PRED_LOAD_CONTINUE9:%.*]] +; FVW2: pred.load.if8: +; FVW2-NEXT: [[TMP41:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], i64 [[TMP0]], i32 1 +; FVW2-NEXT: [[TMP42:%.*]] = load float, float addrspace(1)* [[TMP41]], align 4 +; FVW2-NEXT: [[TMP43:%.*]] = insertelement <2 x float> [[TMP39]], float [[TMP42]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE9]] +; FVW2: pred.load.continue9: +; FVW2-NEXT: [[TMP44:%.*]] = phi <2 x float> [ [[TMP39]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP43]], [[PRED_LOAD_IF8]] ] +; FVW2-NEXT: [[TMP45:%.*]] = extractelement <2 x i1> [[TMP32]], i32 0 +; FVW2-NEXT: br i1 [[TMP45]], label [[PRED_LOAD_IF10:%.*]], label [[PRED_LOAD_CONTINUE11:%.*]] +; FVW2: pred.load.if10: +; FVW2-NEXT: [[TMP46:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], i64 [[TMP1]], i32 1 +; FVW2-NEXT: [[TMP47:%.*]] = load float, float addrspace(1)* [[TMP46]], align 4 +; FVW2-NEXT: [[TMP48:%.*]] = insertelement <2 x float> poison, float [[TMP47]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE11]] +; FVW2: pred.load.continue11: +; FVW2-NEXT: [[TMP49:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE9]] ], [ [[TMP48]], [[PRED_LOAD_IF10]] ] +; FVW2-NEXT: [[TMP50:%.*]] = extractelement <2 x i1> [[TMP32]], i32 1 +; FVW2-NEXT: br i1 [[TMP50]], label [[PRED_LOAD_IF12:%.*]], label [[PRED_LOAD_CONTINUE13:%.*]] +; FVW2: pred.load.if12: +; FVW2-NEXT: [[TMP51:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], i64 [[TMP2]], i32 1 +; FVW2-NEXT: [[TMP52:%.*]] = load float, float addrspace(1)* [[TMP51]], align 4 +; FVW2-NEXT: [[TMP53:%.*]] = insertelement <2 x float> [[TMP49]], float [[TMP52]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE13]] +; FVW2: pred.load.continue13: +; FVW2-NEXT: [[TMP54:%.*]] = phi <2 x float> [ [[TMP49]], [[PRED_LOAD_CONTINUE11]] ], [ [[TMP53]], [[PRED_LOAD_IF12]] ] +; FVW2-NEXT: [[TMP55:%.*]] = extractelement <2 x i1> [[TMP33]], i32 0 +; FVW2-NEXT: br i1 [[TMP55]], label [[PRED_LOAD_IF14:%.*]], label [[PRED_LOAD_CONTINUE15:%.*]] +; FVW2: pred.load.if14: +; FVW2-NEXT: [[TMP56:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], i64 [[TMP3]], i32 1 +; FVW2-NEXT: [[TMP57:%.*]] = load float, float addrspace(1)* [[TMP56]], align 4 +; FVW2-NEXT: [[TMP58:%.*]] = insertelement <2 x float> poison, float [[TMP57]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE15]] +; FVW2: pred.load.continue15: +; FVW2-NEXT: [[TMP59:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE13]] ], [ [[TMP58]], [[PRED_LOAD_IF14]] ] +; FVW2-NEXT: [[TMP60:%.*]] = extractelement <2 x i1> [[TMP33]], i32 1 +; FVW2-NEXT: br i1 [[TMP60]], label [[PRED_LOAD_IF16:%.*]], label [[PRED_LOAD_CONTINUE17:%.*]] +; FVW2: pred.load.if16: +; FVW2-NEXT: [[TMP61:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], i64 [[TMP4]], i32 1 +; FVW2-NEXT: [[TMP62:%.*]] = load float, float addrspace(1)* [[TMP61]], align 4 +; FVW2-NEXT: [[TMP63:%.*]] = insertelement <2 x float> [[TMP59]], float [[TMP62]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE17]] +; FVW2: pred.load.continue17: +; FVW2-NEXT: [[TMP64:%.*]] = phi <2 x float> [ [[TMP59]], [[PRED_LOAD_CONTINUE15]] ], [ [[TMP63]], [[PRED_LOAD_IF16]] ] +; FVW2-NEXT: [[TMP65:%.*]] = extractelement <2 x i1> [[TMP34]], i32 0 +; FVW2-NEXT: br i1 [[TMP65]], label [[PRED_LOAD_IF18:%.*]], label [[PRED_LOAD_CONTINUE19:%.*]] +; FVW2: pred.load.if18: +; FVW2-NEXT: [[TMP66:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], i64 [[TMP5]], i32 1 +; FVW2-NEXT: [[TMP67:%.*]] = load float, float addrspace(1)* [[TMP66]], align 4 +; FVW2-NEXT: [[TMP68:%.*]] = insertelement <2 x float> poison, float [[TMP67]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE19]] +; FVW2: pred.load.continue19: +; FVW2-NEXT: [[TMP69:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE17]] ], [ [[TMP68]], [[PRED_LOAD_IF18]] ] +; FVW2-NEXT: [[TMP70:%.*]] = extractelement <2 x i1> [[TMP34]], i32 1 +; FVW2-NEXT: br i1 [[TMP70]], label [[PRED_LOAD_IF20:%.*]], label [[PRED_LOAD_CONTINUE21:%.*]] +; FVW2: pred.load.if20: +; FVW2-NEXT: [[TMP71:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], i64 [[TMP6]], i32 1 +; FVW2-NEXT: [[TMP72:%.*]] = load float, float addrspace(1)* [[TMP71]], align 4 +; FVW2-NEXT: [[TMP73:%.*]] = insertelement <2 x float> [[TMP69]], float [[TMP72]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE21]] +; FVW2: pred.load.continue21: +; FVW2-NEXT: [[TMP74:%.*]] = phi <2 x float> [ [[TMP69]], [[PRED_LOAD_CONTINUE19]] ], [ [[TMP73]], [[PRED_LOAD_IF20]] ] +; FVW2-NEXT: [[TMP75:%.*]] = fadd <2 x float> [[TMP44]], +; FVW2-NEXT: [[TMP76:%.*]] = fadd <2 x float> [[TMP54]], +; FVW2-NEXT: [[TMP77:%.*]] = fadd <2 x float> [[TMP64]], +; FVW2-NEXT: [[TMP78:%.*]] = fadd <2 x float> [[TMP74]], +; FVW2-NEXT: [[TMP79:%.*]] = extractelement <2 x i1> [[TMP31]], i32 0 +; FVW2-NEXT: br i1 [[TMP79]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] ; FVW2: pred.store.if: -; FVW2-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT:%.*]], i64 [[OFFSET_IDX]] -; FVW2-NEXT: [[TMP12:%.*]] = extractelement <2 x float> [[TMP9]], i32 0 -; FVW2-NEXT: store float [[TMP12]], float addrspace(1)* [[TMP11]], align 4 +; FVW2-NEXT: [[TMP80:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT:%.*]], i64 [[OFFSET_IDX]] +; FVW2-NEXT: [[TMP81:%.*]] = extractelement <2 x float> [[TMP75]], i32 0 +; FVW2-NEXT: store float [[TMP81]], float addrspace(1)* [[TMP80]], align 4 ; FVW2-NEXT: br label [[PRED_STORE_CONTINUE]] ; FVW2: pred.store.continue: -; FVW2-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1 -; FVW2-NEXT: br i1 [[TMP13]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9]] -; FVW2: pred.store.if8: -; FVW2-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP0]] -; FVW2-NEXT: [[TMP15:%.*]] = extractelement <2 x float> [[TMP9]], i32 1 -; FVW2-NEXT: store float [[TMP15]], float addrspace(1)* [[TMP14]], align 4 -; FVW2-NEXT: br label [[PRED_STORE_CONTINUE9]] -; FVW2: pred.store.continue9: -; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX7]], 2 -; FVW2-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], -; FVW2-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 -; FVW2-NEXT: br i1 [[TMP16]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; FVW2-NEXT: [[TMP82:%.*]] = extractelement <2 x i1> [[TMP31]], i32 1 +; FVW2-NEXT: br i1 [[TMP82]], label [[PRED_STORE_IF22:%.*]], label [[PRED_STORE_CONTINUE23:%.*]] +; FVW2: pred.store.if22: +; FVW2-NEXT: [[TMP83:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP0]] +; FVW2-NEXT: [[TMP84:%.*]] = extractelement <2 x float> [[TMP75]], i32 1 +; FVW2-NEXT: store float [[TMP84]], float addrspace(1)* [[TMP83]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE23]] +; FVW2: pred.store.continue23: +; FVW2-NEXT: [[TMP85:%.*]] = extractelement <2 x i1> [[TMP32]], i32 0 +; FVW2-NEXT: br i1 [[TMP85]], label [[PRED_STORE_IF24:%.*]], label [[PRED_STORE_CONTINUE25:%.*]] +; FVW2: pred.store.if24: +; FVW2-NEXT: [[TMP86:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP1]] +; FVW2-NEXT: [[TMP87:%.*]] = extractelement <2 x float> [[TMP76]], i32 0 +; FVW2-NEXT: store float [[TMP87]], float addrspace(1)* [[TMP86]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE25]] +; FVW2: pred.store.continue25: +; FVW2-NEXT: [[TMP88:%.*]] = extractelement <2 x i1> [[TMP32]], i32 1 +; FVW2-NEXT: br i1 [[TMP88]], label [[PRED_STORE_IF26:%.*]], label [[PRED_STORE_CONTINUE27:%.*]] +; FVW2: pred.store.if26: +; FVW2-NEXT: [[TMP89:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP2]] +; FVW2-NEXT: [[TMP90:%.*]] = extractelement <2 x float> [[TMP76]], i32 1 +; FVW2-NEXT: store float [[TMP90]], float addrspace(1)* [[TMP89]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE27]] +; FVW2: pred.store.continue27: +; FVW2-NEXT: [[TMP91:%.*]] = extractelement <2 x i1> [[TMP33]], i32 0 +; FVW2-NEXT: br i1 [[TMP91]], label [[PRED_STORE_IF28:%.*]], label [[PRED_STORE_CONTINUE29:%.*]] +; FVW2: pred.store.if28: +; FVW2-NEXT: [[TMP92:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP3]] +; FVW2-NEXT: [[TMP93:%.*]] = extractelement <2 x float> [[TMP77]], i32 0 +; FVW2-NEXT: store float [[TMP93]], float addrspace(1)* [[TMP92]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE29]] +; FVW2: pred.store.continue29: +; FVW2-NEXT: [[TMP94:%.*]] = extractelement <2 x i1> [[TMP33]], i32 1 +; FVW2-NEXT: br i1 [[TMP94]], label [[PRED_STORE_IF30:%.*]], label [[PRED_STORE_CONTINUE31:%.*]] +; FVW2: pred.store.if30: +; FVW2-NEXT: [[TMP95:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP4]] +; FVW2-NEXT: [[TMP96:%.*]] = extractelement <2 x float> [[TMP77]], i32 1 +; FVW2-NEXT: store float [[TMP96]], float addrspace(1)* [[TMP95]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE31]] +; FVW2: pred.store.continue31: +; FVW2-NEXT: [[TMP97:%.*]] = extractelement <2 x i1> [[TMP34]], i32 0 +; FVW2-NEXT: br i1 [[TMP97]], label [[PRED_STORE_IF32:%.*]], label [[PRED_STORE_CONTINUE33:%.*]] +; FVW2: pred.store.if32: +; FVW2-NEXT: [[TMP98:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP5]] +; FVW2-NEXT: [[TMP99:%.*]] = extractelement <2 x float> [[TMP78]], i32 0 +; FVW2-NEXT: store float [[TMP99]], float addrspace(1)* [[TMP98]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE33]] +; FVW2: pred.store.continue33: +; FVW2-NEXT: [[TMP100:%.*]] = extractelement <2 x i1> [[TMP34]], i32 1 +; FVW2-NEXT: br i1 [[TMP100]], label [[PRED_STORE_IF34:%.*]], label [[PRED_STORE_CONTINUE35]] +; FVW2: pred.store.if34: +; FVW2-NEXT: [[TMP101:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP6]] +; FVW2-NEXT: [[TMP102:%.*]] = extractelement <2 x float> [[TMP78]], i32 1 +; FVW2-NEXT: store float [[TMP102]], float addrspace(1)* [[TMP101]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE35]] +; FVW2: pred.store.continue35: +; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX7]], 8 +; FVW2-NEXT: [[TMP103:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 +; FVW2-NEXT: br i1 [[TMP103]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; FVW2: for.end: ; FVW2-NEXT: ret void ; @@ -1072,40 +1582,186 @@ ; FVW2-NEXT: entry: ; FVW2-NEXT: br label [[VECTOR_BODY:%.*]] ; FVW2: vector.body: -; FVW2-NEXT: [[INDEX7:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE9:%.*]] ] -; FVW2-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ , [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE9]] ] +; FVW2-NEXT: [[INDEX7:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE35:%.*]] ] ; FVW2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX7]], 4 ; FVW2-NEXT: [[TMP0:%.*]] = or i64 [[OFFSET_IDX]], 16 -; FVW2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[OFFSET_IDX]] -; FVW2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP0]] -; FVW2-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1]], align 4 -; FVW2-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP2]], align 4 -; FVW2-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i32 0 -; FVW2-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP4]], i32 1 -; FVW2-NEXT: [[TMP7:%.*]] = icmp sgt <2 x i32> [[TMP6]], zeroinitializer -; FVW2-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], [[STRUCT_IN]] addrspace(1)* [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1 -; FVW2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p1f32(<2 x float addrspace(1)*> [[TMP8]], i32 4, <2 x i1> [[TMP7]], <2 x float> undef) -; FVW2-NEXT: [[TMP9:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER]], -; FVW2-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0 -; FVW2-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] +; FVW2-NEXT: [[TMP1:%.*]] = or i64 [[OFFSET_IDX]], 32 +; FVW2-NEXT: [[TMP2:%.*]] = or i64 [[OFFSET_IDX]], 48 +; FVW2-NEXT: [[TMP3:%.*]] = or i64 [[OFFSET_IDX]], 64 +; FVW2-NEXT: [[TMP4:%.*]] = or i64 [[OFFSET_IDX]], 80 +; FVW2-NEXT: [[TMP5:%.*]] = or i64 [[OFFSET_IDX]], 96 +; FVW2-NEXT: [[TMP6:%.*]] = or i64 [[OFFSET_IDX]], 112 +; FVW2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[OFFSET_IDX]] +; FVW2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP0]] +; FVW2-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP1]] +; FVW2-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP2]] +; FVW2-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP3]] +; FVW2-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP4]] +; FVW2-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP5]] +; FVW2-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP6]] +; FVW2-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP7]], align 4 +; FVW2-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP8]], align 4 +; FVW2-NEXT: [[TMP17:%.*]] = insertelement <2 x i32> poison, i32 [[TMP15]], i32 0 +; FVW2-NEXT: [[TMP18:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP16]], i32 1 +; FVW2-NEXT: [[TMP19:%.*]] = load i32, i32* [[TMP9]], align 4 +; FVW2-NEXT: [[TMP20:%.*]] = load i32, i32* [[TMP10]], align 4 +; FVW2-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> poison, i32 [[TMP19]], i32 0 +; FVW2-NEXT: [[TMP22:%.*]] = insertelement <2 x i32> [[TMP21]], i32 [[TMP20]], i32 1 +; FVW2-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP11]], align 4 +; FVW2-NEXT: [[TMP24:%.*]] = load i32, i32* [[TMP12]], align 4 +; FVW2-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> poison, i32 [[TMP23]], i32 0 +; FVW2-NEXT: [[TMP26:%.*]] = insertelement <2 x i32> [[TMP25]], i32 [[TMP24]], i32 1 +; FVW2-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP13]], align 4 +; FVW2-NEXT: [[TMP28:%.*]] = load i32, i32* [[TMP14]], align 4 +; FVW2-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> poison, i32 [[TMP27]], i32 0 +; FVW2-NEXT: [[TMP30:%.*]] = insertelement <2 x i32> [[TMP29]], i32 [[TMP28]], i32 1 +; FVW2-NEXT: [[TMP31:%.*]] = icmp sgt <2 x i32> [[TMP18]], zeroinitializer +; FVW2-NEXT: [[TMP32:%.*]] = icmp sgt <2 x i32> [[TMP22]], zeroinitializer +; FVW2-NEXT: [[TMP33:%.*]] = icmp sgt <2 x i32> [[TMP26]], zeroinitializer +; FVW2-NEXT: [[TMP34:%.*]] = icmp sgt <2 x i32> [[TMP30]], zeroinitializer +; FVW2-NEXT: [[TMP35:%.*]] = extractelement <2 x i1> [[TMP31]], i32 0 +; FVW2-NEXT: br i1 [[TMP35]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; FVW2: pred.load.if: +; FVW2-NEXT: [[TMP36:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], [[STRUCT_IN]] addrspace(1)* [[IN:%.*]], i64 [[OFFSET_IDX]], i32 1 +; FVW2-NEXT: [[TMP37:%.*]] = load float, float addrspace(1)* [[TMP36]], align 4 +; FVW2-NEXT: [[TMP38:%.*]] = insertelement <2 x float> poison, float [[TMP37]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE]] +; FVW2: pred.load.continue: +; FVW2-NEXT: [[TMP39:%.*]] = phi <2 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP38]], [[PRED_LOAD_IF]] ] +; FVW2-NEXT: [[TMP40:%.*]] = extractelement <2 x i1> [[TMP31]], i32 1 +; FVW2-NEXT: br i1 [[TMP40]], label [[PRED_LOAD_IF8:%.*]], label [[PRED_LOAD_CONTINUE9:%.*]] +; FVW2: pred.load.if8: +; FVW2-NEXT: [[TMP41:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], i64 [[TMP0]], i32 1 +; FVW2-NEXT: [[TMP42:%.*]] = load float, float addrspace(1)* [[TMP41]], align 4 +; FVW2-NEXT: [[TMP43:%.*]] = insertelement <2 x float> [[TMP39]], float [[TMP42]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE9]] +; FVW2: pred.load.continue9: +; FVW2-NEXT: [[TMP44:%.*]] = phi <2 x float> [ [[TMP39]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP43]], [[PRED_LOAD_IF8]] ] +; FVW2-NEXT: [[TMP45:%.*]] = extractelement <2 x i1> [[TMP32]], i32 0 +; FVW2-NEXT: br i1 [[TMP45]], label [[PRED_LOAD_IF10:%.*]], label [[PRED_LOAD_CONTINUE11:%.*]] +; FVW2: pred.load.if10: +; FVW2-NEXT: [[TMP46:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], i64 [[TMP1]], i32 1 +; FVW2-NEXT: [[TMP47:%.*]] = load float, float addrspace(1)* [[TMP46]], align 4 +; FVW2-NEXT: [[TMP48:%.*]] = insertelement <2 x float> poison, float [[TMP47]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE11]] +; FVW2: pred.load.continue11: +; FVW2-NEXT: [[TMP49:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE9]] ], [ [[TMP48]], [[PRED_LOAD_IF10]] ] +; FVW2-NEXT: [[TMP50:%.*]] = extractelement <2 x i1> [[TMP32]], i32 1 +; FVW2-NEXT: br i1 [[TMP50]], label [[PRED_LOAD_IF12:%.*]], label [[PRED_LOAD_CONTINUE13:%.*]] +; FVW2: pred.load.if12: +; FVW2-NEXT: [[TMP51:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], i64 [[TMP2]], i32 1 +; FVW2-NEXT: [[TMP52:%.*]] = load float, float addrspace(1)* [[TMP51]], align 4 +; FVW2-NEXT: [[TMP53:%.*]] = insertelement <2 x float> [[TMP49]], float [[TMP52]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE13]] +; FVW2: pred.load.continue13: +; FVW2-NEXT: [[TMP54:%.*]] = phi <2 x float> [ [[TMP49]], [[PRED_LOAD_CONTINUE11]] ], [ [[TMP53]], [[PRED_LOAD_IF12]] ] +; FVW2-NEXT: [[TMP55:%.*]] = extractelement <2 x i1> [[TMP33]], i32 0 +; FVW2-NEXT: br i1 [[TMP55]], label [[PRED_LOAD_IF14:%.*]], label [[PRED_LOAD_CONTINUE15:%.*]] +; FVW2: pred.load.if14: +; FVW2-NEXT: [[TMP56:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], i64 [[TMP3]], i32 1 +; FVW2-NEXT: [[TMP57:%.*]] = load float, float addrspace(1)* [[TMP56]], align 4 +; FVW2-NEXT: [[TMP58:%.*]] = insertelement <2 x float> poison, float [[TMP57]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE15]] +; FVW2: pred.load.continue15: +; FVW2-NEXT: [[TMP59:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE13]] ], [ [[TMP58]], [[PRED_LOAD_IF14]] ] +; FVW2-NEXT: [[TMP60:%.*]] = extractelement <2 x i1> [[TMP33]], i32 1 +; FVW2-NEXT: br i1 [[TMP60]], label [[PRED_LOAD_IF16:%.*]], label [[PRED_LOAD_CONTINUE17:%.*]] +; FVW2: pred.load.if16: +; FVW2-NEXT: [[TMP61:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], i64 [[TMP4]], i32 1 +; FVW2-NEXT: [[TMP62:%.*]] = load float, float addrspace(1)* [[TMP61]], align 4 +; FVW2-NEXT: [[TMP63:%.*]] = insertelement <2 x float> [[TMP59]], float [[TMP62]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE17]] +; FVW2: pred.load.continue17: +; FVW2-NEXT: [[TMP64:%.*]] = phi <2 x float> [ [[TMP59]], [[PRED_LOAD_CONTINUE15]] ], [ [[TMP63]], [[PRED_LOAD_IF16]] ] +; FVW2-NEXT: [[TMP65:%.*]] = extractelement <2 x i1> [[TMP34]], i32 0 +; FVW2-NEXT: br i1 [[TMP65]], label [[PRED_LOAD_IF18:%.*]], label [[PRED_LOAD_CONTINUE19:%.*]] +; FVW2: pred.load.if18: +; FVW2-NEXT: [[TMP66:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], i64 [[TMP5]], i32 1 +; FVW2-NEXT: [[TMP67:%.*]] = load float, float addrspace(1)* [[TMP66]], align 4 +; FVW2-NEXT: [[TMP68:%.*]] = insertelement <2 x float> poison, float [[TMP67]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE19]] +; FVW2: pred.load.continue19: +; FVW2-NEXT: [[TMP69:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE17]] ], [ [[TMP68]], [[PRED_LOAD_IF18]] ] +; FVW2-NEXT: [[TMP70:%.*]] = extractelement <2 x i1> [[TMP34]], i32 1 +; FVW2-NEXT: br i1 [[TMP70]], label [[PRED_LOAD_IF20:%.*]], label [[PRED_LOAD_CONTINUE21:%.*]] +; FVW2: pred.load.if20: +; FVW2-NEXT: [[TMP71:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], i64 [[TMP6]], i32 1 +; FVW2-NEXT: [[TMP72:%.*]] = load float, float addrspace(1)* [[TMP71]], align 4 +; FVW2-NEXT: [[TMP73:%.*]] = insertelement <2 x float> [[TMP69]], float [[TMP72]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE21]] +; FVW2: pred.load.continue21: +; FVW2-NEXT: [[TMP74:%.*]] = phi <2 x float> [ [[TMP69]], [[PRED_LOAD_CONTINUE19]] ], [ [[TMP73]], [[PRED_LOAD_IF20]] ] +; FVW2-NEXT: [[TMP75:%.*]] = fadd <2 x float> [[TMP44]], +; FVW2-NEXT: [[TMP76:%.*]] = fadd <2 x float> [[TMP54]], +; FVW2-NEXT: [[TMP77:%.*]] = fadd <2 x float> [[TMP64]], +; FVW2-NEXT: [[TMP78:%.*]] = fadd <2 x float> [[TMP74]], +; FVW2-NEXT: [[TMP79:%.*]] = extractelement <2 x i1> [[TMP31]], i32 0 +; FVW2-NEXT: br i1 [[TMP79]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] ; FVW2: pred.store.if: -; FVW2-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, float* [[OUT:%.*]], i64 [[OFFSET_IDX]] -; FVW2-NEXT: [[TMP12:%.*]] = extractelement <2 x float> [[TMP9]], i32 0 -; FVW2-NEXT: store float [[TMP12]], float* [[TMP11]], align 4 +; FVW2-NEXT: [[TMP80:%.*]] = getelementptr inbounds float, float* [[OUT:%.*]], i64 [[OFFSET_IDX]] +; FVW2-NEXT: [[TMP81:%.*]] = extractelement <2 x float> [[TMP75]], i32 0 +; FVW2-NEXT: store float [[TMP81]], float* [[TMP80]], align 4 ; FVW2-NEXT: br label [[PRED_STORE_CONTINUE]] ; FVW2: pred.store.continue: -; FVW2-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1 -; FVW2-NEXT: br i1 [[TMP13]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9]] -; FVW2: pred.store.if8: -; FVW2-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP0]] -; FVW2-NEXT: [[TMP15:%.*]] = extractelement <2 x float> [[TMP9]], i32 1 -; FVW2-NEXT: store float [[TMP15]], float* [[TMP14]], align 4 -; FVW2-NEXT: br label [[PRED_STORE_CONTINUE9]] -; FVW2: pred.store.continue9: -; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX7]], 2 -; FVW2-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], -; FVW2-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 -; FVW2-NEXT: br i1 [[TMP16]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; FVW2-NEXT: [[TMP82:%.*]] = extractelement <2 x i1> [[TMP31]], i32 1 +; FVW2-NEXT: br i1 [[TMP82]], label [[PRED_STORE_IF22:%.*]], label [[PRED_STORE_CONTINUE23:%.*]] +; FVW2: pred.store.if22: +; FVW2-NEXT: [[TMP83:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP0]] +; FVW2-NEXT: [[TMP84:%.*]] = extractelement <2 x float> [[TMP75]], i32 1 +; FVW2-NEXT: store float [[TMP84]], float* [[TMP83]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE23]] +; FVW2: pred.store.continue23: +; FVW2-NEXT: [[TMP85:%.*]] = extractelement <2 x i1> [[TMP32]], i32 0 +; FVW2-NEXT: br i1 [[TMP85]], label [[PRED_STORE_IF24:%.*]], label [[PRED_STORE_CONTINUE25:%.*]] +; FVW2: pred.store.if24: +; FVW2-NEXT: [[TMP86:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP1]] +; FVW2-NEXT: [[TMP87:%.*]] = extractelement <2 x float> [[TMP76]], i32 0 +; FVW2-NEXT: store float [[TMP87]], float* [[TMP86]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE25]] +; FVW2: pred.store.continue25: +; FVW2-NEXT: [[TMP88:%.*]] = extractelement <2 x i1> [[TMP32]], i32 1 +; FVW2-NEXT: br i1 [[TMP88]], label [[PRED_STORE_IF26:%.*]], label [[PRED_STORE_CONTINUE27:%.*]] +; FVW2: pred.store.if26: +; FVW2-NEXT: [[TMP89:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP2]] +; FVW2-NEXT: [[TMP90:%.*]] = extractelement <2 x float> [[TMP76]], i32 1 +; FVW2-NEXT: store float [[TMP90]], float* [[TMP89]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE27]] +; FVW2: pred.store.continue27: +; FVW2-NEXT: [[TMP91:%.*]] = extractelement <2 x i1> [[TMP33]], i32 0 +; FVW2-NEXT: br i1 [[TMP91]], label [[PRED_STORE_IF28:%.*]], label [[PRED_STORE_CONTINUE29:%.*]] +; FVW2: pred.store.if28: +; FVW2-NEXT: [[TMP92:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP3]] +; FVW2-NEXT: [[TMP93:%.*]] = extractelement <2 x float> [[TMP77]], i32 0 +; FVW2-NEXT: store float [[TMP93]], float* [[TMP92]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE29]] +; FVW2: pred.store.continue29: +; FVW2-NEXT: [[TMP94:%.*]] = extractelement <2 x i1> [[TMP33]], i32 1 +; FVW2-NEXT: br i1 [[TMP94]], label [[PRED_STORE_IF30:%.*]], label [[PRED_STORE_CONTINUE31:%.*]] +; FVW2: pred.store.if30: +; FVW2-NEXT: [[TMP95:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP4]] +; FVW2-NEXT: [[TMP96:%.*]] = extractelement <2 x float> [[TMP77]], i32 1 +; FVW2-NEXT: store float [[TMP96]], float* [[TMP95]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE31]] +; FVW2: pred.store.continue31: +; FVW2-NEXT: [[TMP97:%.*]] = extractelement <2 x i1> [[TMP34]], i32 0 +; FVW2-NEXT: br i1 [[TMP97]], label [[PRED_STORE_IF32:%.*]], label [[PRED_STORE_CONTINUE33:%.*]] +; FVW2: pred.store.if32: +; FVW2-NEXT: [[TMP98:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP5]] +; FVW2-NEXT: [[TMP99:%.*]] = extractelement <2 x float> [[TMP78]], i32 0 +; FVW2-NEXT: store float [[TMP99]], float* [[TMP98]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE33]] +; FVW2: pred.store.continue33: +; FVW2-NEXT: [[TMP100:%.*]] = extractelement <2 x i1> [[TMP34]], i32 1 +; FVW2-NEXT: br i1 [[TMP100]], label [[PRED_STORE_IF34:%.*]], label [[PRED_STORE_CONTINUE35]] +; FVW2: pred.store.if34: +; FVW2-NEXT: [[TMP101:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP6]] +; FVW2-NEXT: [[TMP102:%.*]] = extractelement <2 x float> [[TMP78]], i32 1 +; FVW2-NEXT: store float [[TMP102]], float* [[TMP101]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE35]] +; FVW2: pred.store.continue35: +; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX7]], 8 +; FVW2-NEXT: [[TMP103:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 +; FVW2-NEXT: br i1 [[TMP103]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; FVW2: for.end: ; FVW2-NEXT: ret void ; @@ -1303,40 +1959,186 @@ ; FVW2-NEXT: entry: ; FVW2-NEXT: br label [[VECTOR_BODY:%.*]] ; FVW2: vector.body: -; FVW2-NEXT: [[INDEX7:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE9:%.*]] ] -; FVW2-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ , [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE9]] ] +; FVW2-NEXT: [[INDEX7:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE35:%.*]] ] ; FVW2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX7]], 4 ; FVW2-NEXT: [[TMP0:%.*]] = or i64 [[OFFSET_IDX]], 16 -; FVW2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[OFFSET_IDX]] -; FVW2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP0]] -; FVW2-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1]], align 4 -; FVW2-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP2]], align 4 -; FVW2-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i32 0 -; FVW2-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP4]], i32 1 -; FVW2-NEXT: [[TMP7:%.*]] = icmp sgt <2 x i32> [[TMP6]], zeroinitializer -; FVW2-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], %struct.In* [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1 -; FVW2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP8]], i32 4, <2 x i1> [[TMP7]], <2 x float> undef) -; FVW2-NEXT: [[TMP9:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER]], -; FVW2-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0 -; FVW2-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] +; FVW2-NEXT: [[TMP1:%.*]] = or i64 [[OFFSET_IDX]], 32 +; FVW2-NEXT: [[TMP2:%.*]] = or i64 [[OFFSET_IDX]], 48 +; FVW2-NEXT: [[TMP3:%.*]] = or i64 [[OFFSET_IDX]], 64 +; FVW2-NEXT: [[TMP4:%.*]] = or i64 [[OFFSET_IDX]], 80 +; FVW2-NEXT: [[TMP5:%.*]] = or i64 [[OFFSET_IDX]], 96 +; FVW2-NEXT: [[TMP6:%.*]] = or i64 [[OFFSET_IDX]], 112 +; FVW2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[OFFSET_IDX]] +; FVW2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP0]] +; FVW2-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP1]] +; FVW2-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP2]] +; FVW2-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP3]] +; FVW2-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP4]] +; FVW2-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP5]] +; FVW2-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP6]] +; FVW2-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP7]], align 4 +; FVW2-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP8]], align 4 +; FVW2-NEXT: [[TMP17:%.*]] = insertelement <2 x i32> poison, i32 [[TMP15]], i32 0 +; FVW2-NEXT: [[TMP18:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP16]], i32 1 +; FVW2-NEXT: [[TMP19:%.*]] = load i32, i32* [[TMP9]], align 4 +; FVW2-NEXT: [[TMP20:%.*]] = load i32, i32* [[TMP10]], align 4 +; FVW2-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> poison, i32 [[TMP19]], i32 0 +; FVW2-NEXT: [[TMP22:%.*]] = insertelement <2 x i32> [[TMP21]], i32 [[TMP20]], i32 1 +; FVW2-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP11]], align 4 +; FVW2-NEXT: [[TMP24:%.*]] = load i32, i32* [[TMP12]], align 4 +; FVW2-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> poison, i32 [[TMP23]], i32 0 +; FVW2-NEXT: [[TMP26:%.*]] = insertelement <2 x i32> [[TMP25]], i32 [[TMP24]], i32 1 +; FVW2-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP13]], align 4 +; FVW2-NEXT: [[TMP28:%.*]] = load i32, i32* [[TMP14]], align 4 +; FVW2-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> poison, i32 [[TMP27]], i32 0 +; FVW2-NEXT: [[TMP30:%.*]] = insertelement <2 x i32> [[TMP29]], i32 [[TMP28]], i32 1 +; FVW2-NEXT: [[TMP31:%.*]] = icmp sgt <2 x i32> [[TMP18]], zeroinitializer +; FVW2-NEXT: [[TMP32:%.*]] = icmp sgt <2 x i32> [[TMP22]], zeroinitializer +; FVW2-NEXT: [[TMP33:%.*]] = icmp sgt <2 x i32> [[TMP26]], zeroinitializer +; FVW2-NEXT: [[TMP34:%.*]] = icmp sgt <2 x i32> [[TMP30]], zeroinitializer +; FVW2-NEXT: [[TMP35:%.*]] = extractelement <2 x i1> [[TMP31]], i32 0 +; FVW2-NEXT: br i1 [[TMP35]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; FVW2: pred.load.if: +; FVW2-NEXT: [[TMP36:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], %struct.In* [[IN:%.*]], i64 [[OFFSET_IDX]], i32 1 +; FVW2-NEXT: [[TMP37:%.*]] = load float, float* [[TMP36]], align 4 +; FVW2-NEXT: [[TMP38:%.*]] = insertelement <2 x float> poison, float [[TMP37]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE]] +; FVW2: pred.load.continue: +; FVW2-NEXT: [[TMP39:%.*]] = phi <2 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP38]], [[PRED_LOAD_IF]] ] +; FVW2-NEXT: [[TMP40:%.*]] = extractelement <2 x i1> [[TMP31]], i32 1 +; FVW2-NEXT: br i1 [[TMP40]], label [[PRED_LOAD_IF8:%.*]], label [[PRED_LOAD_CONTINUE9:%.*]] +; FVW2: pred.load.if8: +; FVW2-NEXT: [[TMP41:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP0]], i32 1 +; FVW2-NEXT: [[TMP42:%.*]] = load float, float* [[TMP41]], align 4 +; FVW2-NEXT: [[TMP43:%.*]] = insertelement <2 x float> [[TMP39]], float [[TMP42]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE9]] +; FVW2: pred.load.continue9: +; FVW2-NEXT: [[TMP44:%.*]] = phi <2 x float> [ [[TMP39]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP43]], [[PRED_LOAD_IF8]] ] +; FVW2-NEXT: [[TMP45:%.*]] = extractelement <2 x i1> [[TMP32]], i32 0 +; FVW2-NEXT: br i1 [[TMP45]], label [[PRED_LOAD_IF10:%.*]], label [[PRED_LOAD_CONTINUE11:%.*]] +; FVW2: pred.load.if10: +; FVW2-NEXT: [[TMP46:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP1]], i32 1 +; FVW2-NEXT: [[TMP47:%.*]] = load float, float* [[TMP46]], align 4 +; FVW2-NEXT: [[TMP48:%.*]] = insertelement <2 x float> poison, float [[TMP47]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE11]] +; FVW2: pred.load.continue11: +; FVW2-NEXT: [[TMP49:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE9]] ], [ [[TMP48]], [[PRED_LOAD_IF10]] ] +; FVW2-NEXT: [[TMP50:%.*]] = extractelement <2 x i1> [[TMP32]], i32 1 +; FVW2-NEXT: br i1 [[TMP50]], label [[PRED_LOAD_IF12:%.*]], label [[PRED_LOAD_CONTINUE13:%.*]] +; FVW2: pred.load.if12: +; FVW2-NEXT: [[TMP51:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP2]], i32 1 +; FVW2-NEXT: [[TMP52:%.*]] = load float, float* [[TMP51]], align 4 +; FVW2-NEXT: [[TMP53:%.*]] = insertelement <2 x float> [[TMP49]], float [[TMP52]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE13]] +; FVW2: pred.load.continue13: +; FVW2-NEXT: [[TMP54:%.*]] = phi <2 x float> [ [[TMP49]], [[PRED_LOAD_CONTINUE11]] ], [ [[TMP53]], [[PRED_LOAD_IF12]] ] +; FVW2-NEXT: [[TMP55:%.*]] = extractelement <2 x i1> [[TMP33]], i32 0 +; FVW2-NEXT: br i1 [[TMP55]], label [[PRED_LOAD_IF14:%.*]], label [[PRED_LOAD_CONTINUE15:%.*]] +; FVW2: pred.load.if14: +; FVW2-NEXT: [[TMP56:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP3]], i32 1 +; FVW2-NEXT: [[TMP57:%.*]] = load float, float* [[TMP56]], align 4 +; FVW2-NEXT: [[TMP58:%.*]] = insertelement <2 x float> poison, float [[TMP57]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE15]] +; FVW2: pred.load.continue15: +; FVW2-NEXT: [[TMP59:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE13]] ], [ [[TMP58]], [[PRED_LOAD_IF14]] ] +; FVW2-NEXT: [[TMP60:%.*]] = extractelement <2 x i1> [[TMP33]], i32 1 +; FVW2-NEXT: br i1 [[TMP60]], label [[PRED_LOAD_IF16:%.*]], label [[PRED_LOAD_CONTINUE17:%.*]] +; FVW2: pred.load.if16: +; FVW2-NEXT: [[TMP61:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP4]], i32 1 +; FVW2-NEXT: [[TMP62:%.*]] = load float, float* [[TMP61]], align 4 +; FVW2-NEXT: [[TMP63:%.*]] = insertelement <2 x float> [[TMP59]], float [[TMP62]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE17]] +; FVW2: pred.load.continue17: +; FVW2-NEXT: [[TMP64:%.*]] = phi <2 x float> [ [[TMP59]], [[PRED_LOAD_CONTINUE15]] ], [ [[TMP63]], [[PRED_LOAD_IF16]] ] +; FVW2-NEXT: [[TMP65:%.*]] = extractelement <2 x i1> [[TMP34]], i32 0 +; FVW2-NEXT: br i1 [[TMP65]], label [[PRED_LOAD_IF18:%.*]], label [[PRED_LOAD_CONTINUE19:%.*]] +; FVW2: pred.load.if18: +; FVW2-NEXT: [[TMP66:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP5]], i32 1 +; FVW2-NEXT: [[TMP67:%.*]] = load float, float* [[TMP66]], align 4 +; FVW2-NEXT: [[TMP68:%.*]] = insertelement <2 x float> poison, float [[TMP67]], i32 0 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE19]] +; FVW2: pred.load.continue19: +; FVW2-NEXT: [[TMP69:%.*]] = phi <2 x float> [ poison, [[PRED_LOAD_CONTINUE17]] ], [ [[TMP68]], [[PRED_LOAD_IF18]] ] +; FVW2-NEXT: [[TMP70:%.*]] = extractelement <2 x i1> [[TMP34]], i32 1 +; FVW2-NEXT: br i1 [[TMP70]], label [[PRED_LOAD_IF20:%.*]], label [[PRED_LOAD_CONTINUE21:%.*]] +; FVW2: pred.load.if20: +; FVW2-NEXT: [[TMP71:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], i64 [[TMP6]], i32 1 +; FVW2-NEXT: [[TMP72:%.*]] = load float, float* [[TMP71]], align 4 +; FVW2-NEXT: [[TMP73:%.*]] = insertelement <2 x float> [[TMP69]], float [[TMP72]], i32 1 +; FVW2-NEXT: br label [[PRED_LOAD_CONTINUE21]] +; FVW2: pred.load.continue21: +; FVW2-NEXT: [[TMP74:%.*]] = phi <2 x float> [ [[TMP69]], [[PRED_LOAD_CONTINUE19]] ], [ [[TMP73]], [[PRED_LOAD_IF20]] ] +; FVW2-NEXT: [[TMP75:%.*]] = fadd <2 x float> [[TMP44]], +; FVW2-NEXT: [[TMP76:%.*]] = fadd <2 x float> [[TMP54]], +; FVW2-NEXT: [[TMP77:%.*]] = fadd <2 x float> [[TMP64]], +; FVW2-NEXT: [[TMP78:%.*]] = fadd <2 x float> [[TMP74]], +; FVW2-NEXT: [[TMP79:%.*]] = extractelement <2 x i1> [[TMP31]], i32 0 +; FVW2-NEXT: br i1 [[TMP79]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] ; FVW2: pred.store.if: -; FVW2-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT:%.*]], i64 [[OFFSET_IDX]] -; FVW2-NEXT: [[TMP12:%.*]] = extractelement <2 x float> [[TMP9]], i32 0 -; FVW2-NEXT: store float [[TMP12]], float addrspace(1)* [[TMP11]], align 4 +; FVW2-NEXT: [[TMP80:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT:%.*]], i64 [[OFFSET_IDX]] +; FVW2-NEXT: [[TMP81:%.*]] = extractelement <2 x float> [[TMP75]], i32 0 +; FVW2-NEXT: store float [[TMP81]], float addrspace(1)* [[TMP80]], align 4 ; FVW2-NEXT: br label [[PRED_STORE_CONTINUE]] ; FVW2: pred.store.continue: -; FVW2-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1 -; FVW2-NEXT: br i1 [[TMP13]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9]] -; FVW2: pred.store.if8: -; FVW2-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP0]] -; FVW2-NEXT: [[TMP15:%.*]] = extractelement <2 x float> [[TMP9]], i32 1 -; FVW2-NEXT: store float [[TMP15]], float addrspace(1)* [[TMP14]], align 4 -; FVW2-NEXT: br label [[PRED_STORE_CONTINUE9]] -; FVW2: pred.store.continue9: -; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX7]], 2 -; FVW2-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], -; FVW2-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 -; FVW2-NEXT: br i1 [[TMP16]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; FVW2-NEXT: [[TMP82:%.*]] = extractelement <2 x i1> [[TMP31]], i32 1 +; FVW2-NEXT: br i1 [[TMP82]], label [[PRED_STORE_IF22:%.*]], label [[PRED_STORE_CONTINUE23:%.*]] +; FVW2: pred.store.if22: +; FVW2-NEXT: [[TMP83:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP0]] +; FVW2-NEXT: [[TMP84:%.*]] = extractelement <2 x float> [[TMP75]], i32 1 +; FVW2-NEXT: store float [[TMP84]], float addrspace(1)* [[TMP83]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE23]] +; FVW2: pred.store.continue23: +; FVW2-NEXT: [[TMP85:%.*]] = extractelement <2 x i1> [[TMP32]], i32 0 +; FVW2-NEXT: br i1 [[TMP85]], label [[PRED_STORE_IF24:%.*]], label [[PRED_STORE_CONTINUE25:%.*]] +; FVW2: pred.store.if24: +; FVW2-NEXT: [[TMP86:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP1]] +; FVW2-NEXT: [[TMP87:%.*]] = extractelement <2 x float> [[TMP76]], i32 0 +; FVW2-NEXT: store float [[TMP87]], float addrspace(1)* [[TMP86]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE25]] +; FVW2: pred.store.continue25: +; FVW2-NEXT: [[TMP88:%.*]] = extractelement <2 x i1> [[TMP32]], i32 1 +; FVW2-NEXT: br i1 [[TMP88]], label [[PRED_STORE_IF26:%.*]], label [[PRED_STORE_CONTINUE27:%.*]] +; FVW2: pred.store.if26: +; FVW2-NEXT: [[TMP89:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP2]] +; FVW2-NEXT: [[TMP90:%.*]] = extractelement <2 x float> [[TMP76]], i32 1 +; FVW2-NEXT: store float [[TMP90]], float addrspace(1)* [[TMP89]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE27]] +; FVW2: pred.store.continue27: +; FVW2-NEXT: [[TMP91:%.*]] = extractelement <2 x i1> [[TMP33]], i32 0 +; FVW2-NEXT: br i1 [[TMP91]], label [[PRED_STORE_IF28:%.*]], label [[PRED_STORE_CONTINUE29:%.*]] +; FVW2: pred.store.if28: +; FVW2-NEXT: [[TMP92:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP3]] +; FVW2-NEXT: [[TMP93:%.*]] = extractelement <2 x float> [[TMP77]], i32 0 +; FVW2-NEXT: store float [[TMP93]], float addrspace(1)* [[TMP92]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE29]] +; FVW2: pred.store.continue29: +; FVW2-NEXT: [[TMP94:%.*]] = extractelement <2 x i1> [[TMP33]], i32 1 +; FVW2-NEXT: br i1 [[TMP94]], label [[PRED_STORE_IF30:%.*]], label [[PRED_STORE_CONTINUE31:%.*]] +; FVW2: pred.store.if30: +; FVW2-NEXT: [[TMP95:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP4]] +; FVW2-NEXT: [[TMP96:%.*]] = extractelement <2 x float> [[TMP77]], i32 1 +; FVW2-NEXT: store float [[TMP96]], float addrspace(1)* [[TMP95]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE31]] +; FVW2: pred.store.continue31: +; FVW2-NEXT: [[TMP97:%.*]] = extractelement <2 x i1> [[TMP34]], i32 0 +; FVW2-NEXT: br i1 [[TMP97]], label [[PRED_STORE_IF32:%.*]], label [[PRED_STORE_CONTINUE33:%.*]] +; FVW2: pred.store.if32: +; FVW2-NEXT: [[TMP98:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP5]] +; FVW2-NEXT: [[TMP99:%.*]] = extractelement <2 x float> [[TMP78]], i32 0 +; FVW2-NEXT: store float [[TMP99]], float addrspace(1)* [[TMP98]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE33]] +; FVW2: pred.store.continue33: +; FVW2-NEXT: [[TMP100:%.*]] = extractelement <2 x i1> [[TMP34]], i32 1 +; FVW2-NEXT: br i1 [[TMP100]], label [[PRED_STORE_IF34:%.*]], label [[PRED_STORE_CONTINUE35]] +; FVW2: pred.store.if34: +; FVW2-NEXT: [[TMP101:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP6]] +; FVW2-NEXT: [[TMP102:%.*]] = extractelement <2 x float> [[TMP78]], i32 1 +; FVW2-NEXT: store float [[TMP102]], float addrspace(1)* [[TMP101]], align 4 +; FVW2-NEXT: br label [[PRED_STORE_CONTINUE35]] +; FVW2: pred.store.continue35: +; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX7]], 8 +; FVW2-NEXT: [[TMP103:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256 +; FVW2-NEXT: br i1 [[TMP103]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; FVW2: for.end: ; FVW2-NEXT: ret void ; diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll --- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll @@ -1103,8 +1103,8 @@ ; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer ; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; DISABLED_MASKED_STRIDED: vector.body: -; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE60:%.*]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ , [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE60]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE44:%.*]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ , [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE44]] ] ; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] ; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], ; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP0]], i32 0 @@ -1277,148 +1277,115 @@ ; DISABLED_MASKED_STRIDED-NEXT: [[TMP103:%.*]] = getelementptr inbounds i8, i8* [[Q:%.*]], i32 [[TMP102]] ; DISABLED_MASKED_STRIDED-NEXT: [[TMP104:%.*]] = extractelement <8 x i8> [[TMP100]], i32 0 ; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP104]], i8* [[TMP103]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP105:%.*]] = extractelement <8 x i8> [[TMP100]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = sub i8 0, [[TMP105]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = extractelement <8 x i32> [[TMP50]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP107]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP106]], i8* [[TMP108]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE]] ; DISABLED_MASKED_STRIDED: pred.store.continue: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP105:%.*]] = extractelement <8 x i1> [[TMP0]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP105]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE32:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = extractelement <8 x i1> [[TMP0]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP109]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE32:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if31: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = extractelement <8 x i32> [[TMP1]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP106]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = extractelement <8 x i8> [[TMP100]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP108]], i8* [[TMP107]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = extractelement <8 x i32> [[TMP1]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP110]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = extractelement <8 x i8> [[TMP100]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP112]], i8* [[TMP111]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = extractelement <8 x i8> [[TMP100]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = sub i8 0, [[TMP113]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = extractelement <8 x i32> [[TMP50]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP115]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP114]], i8* [[TMP116]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE32]] ; DISABLED_MASKED_STRIDED: pred.store.continue32: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = extractelement <8 x i1> [[TMP0]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP109]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = extractelement <8 x i1> [[TMP0]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP117]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if33: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = extractelement <8 x i32> [[TMP1]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP110]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = extractelement <8 x i8> [[TMP100]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP112]], i8* [[TMP111]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = extractelement <8 x i32> [[TMP1]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP118]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = extractelement <8 x i8> [[TMP100]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP120]], i8* [[TMP119]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = extractelement <8 x i8> [[TMP100]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = sub i8 0, [[TMP121]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = extractelement <8 x i32> [[TMP50]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP123]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP122]], i8* [[TMP124]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE34]] ; DISABLED_MASKED_STRIDED: pred.store.continue34: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = extractelement <8 x i1> [[TMP0]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP113]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = extractelement <8 x i1> [[TMP0]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP125]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if35: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = extractelement <8 x i32> [[TMP1]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP114]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = extractelement <8 x i8> [[TMP100]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP116]], i8* [[TMP115]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = extractelement <8 x i32> [[TMP1]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP126]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = extractelement <8 x i8> [[TMP100]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP128]], i8* [[TMP127]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = extractelement <8 x i8> [[TMP100]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = sub i8 0, [[TMP129]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = extractelement <8 x i32> [[TMP50]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP131]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP130]], i8* [[TMP132]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE36]] ; DISABLED_MASKED_STRIDED: pred.store.continue36: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = extractelement <8 x i1> [[TMP0]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP117]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = extractelement <8 x i1> [[TMP0]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP133]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if37: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = extractelement <8 x i32> [[TMP1]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP118]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = extractelement <8 x i8> [[TMP100]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP120]], i8* [[TMP119]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = extractelement <8 x i32> [[TMP1]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP134]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i8> [[TMP100]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP136]], i8* [[TMP135]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = extractelement <8 x i8> [[TMP100]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = sub i8 0, [[TMP137]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = extractelement <8 x i32> [[TMP50]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP139]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP138]], i8* [[TMP140]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE38]] ; DISABLED_MASKED_STRIDED: pred.store.continue38: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = extractelement <8 x i1> [[TMP0]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP121]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i1> [[TMP0]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP141]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if39: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = extractelement <8 x i32> [[TMP1]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP122]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = extractelement <8 x i8> [[TMP100]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP124]], i8* [[TMP123]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i32> [[TMP1]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP142]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i8> [[TMP100]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP144]], i8* [[TMP143]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = extractelement <8 x i8> [[TMP100]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = sub i8 0, [[TMP145]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = extractelement <8 x i32> [[TMP50]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP147]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP146]], i8* [[TMP148]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE40]] ; DISABLED_MASKED_STRIDED: pred.store.continue40: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = extractelement <8 x i1> [[TMP0]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP125]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = extractelement <8 x i1> [[TMP0]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP149]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if41: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = extractelement <8 x i32> [[TMP1]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP126]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = extractelement <8 x i8> [[TMP100]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP128]], i8* [[TMP127]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i32> [[TMP1]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP150]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = extractelement <8 x i8> [[TMP100]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP152]], i8* [[TMP151]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = extractelement <8 x i8> [[TMP100]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = sub i8 0, [[TMP153]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = extractelement <8 x i32> [[TMP50]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP155]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP154]], i8* [[TMP156]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE42]] ; DISABLED_MASKED_STRIDED: pred.store.continue42: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = extractelement <8 x i1> [[TMP0]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP129]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = extractelement <8 x i1> [[TMP0]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP157]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44]] ; DISABLED_MASKED_STRIDED: pred.store.if43: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = extractelement <8 x i32> [[TMP1]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP130]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = extractelement <8 x i8> [[TMP100]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP132]], i8* [[TMP131]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE44]] -; DISABLED_MASKED_STRIDED: pred.store.continue44: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = sub <8 x i8> zeroinitializer, [[TMP100]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = extractelement <8 x i1> [[TMP0]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP134]], label [[PRED_STORE_IF45:%.*]], label [[PRED_STORE_CONTINUE46:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if45: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = extractelement <8 x i32> [[TMP50]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP135]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = extractelement <8 x i8> [[TMP133]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP137]], i8* [[TMP136]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE46]] -; DISABLED_MASKED_STRIDED: pred.store.continue46: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i1> [[TMP0]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP138]], label [[PRED_STORE_IF47:%.*]], label [[PRED_STORE_CONTINUE48:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if47: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = extractelement <8 x i32> [[TMP50]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP139]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i8> [[TMP133]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP141]], i8* [[TMP140]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE48]] -; DISABLED_MASKED_STRIDED: pred.store.continue48: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i1> [[TMP0]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP142]], label [[PRED_STORE_IF49:%.*]], label [[PRED_STORE_CONTINUE50:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if49: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = extractelement <8 x i32> [[TMP50]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP143]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = extractelement <8 x i8> [[TMP133]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP145]], i8* [[TMP144]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE50]] -; DISABLED_MASKED_STRIDED: pred.store.continue50: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = extractelement <8 x i1> [[TMP0]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP146]], label [[PRED_STORE_IF51:%.*]], label [[PRED_STORE_CONTINUE52:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if51: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = extractelement <8 x i32> [[TMP50]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP147]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = extractelement <8 x i8> [[TMP133]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP149]], i8* [[TMP148]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE52]] -; DISABLED_MASKED_STRIDED: pred.store.continue52: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i1> [[TMP0]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP150]], label [[PRED_STORE_IF53:%.*]], label [[PRED_STORE_CONTINUE54:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if53: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = extractelement <8 x i32> [[TMP50]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP151]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = extractelement <8 x i8> [[TMP133]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP153]], i8* [[TMP152]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE54]] -; DISABLED_MASKED_STRIDED: pred.store.continue54: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i1> [[TMP0]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP154]], label [[PRED_STORE_IF55:%.*]], label [[PRED_STORE_CONTINUE56:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if55: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = extractelement <8 x i32> [[TMP50]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP155]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = extractelement <8 x i8> [[TMP133]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP157]], i8* [[TMP156]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE56]] -; DISABLED_MASKED_STRIDED: pred.store.continue56: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = extractelement <8 x i1> [[TMP0]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP158]], label [[PRED_STORE_IF57:%.*]], label [[PRED_STORE_CONTINUE58:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if57: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = extractelement <8 x i32> [[TMP50]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP159]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP161:%.*]] = extractelement <8 x i8> [[TMP133]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP161]], i8* [[TMP160]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE58]] -; DISABLED_MASKED_STRIDED: pred.store.continue58: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = extractelement <8 x i1> [[TMP0]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP162]], label [[PRED_STORE_IF59:%.*]], label [[PRED_STORE_CONTINUE60]] -; DISABLED_MASKED_STRIDED: pred.store.if59: +; DISABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = extractelement <8 x i32> [[TMP1]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP158]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = extractelement <8 x i8> [[TMP100]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP160]], i8* [[TMP159]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP161:%.*]] = extractelement <8 x i8> [[TMP100]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = sub i8 0, [[TMP161]] ; DISABLED_MASKED_STRIDED-NEXT: [[TMP163:%.*]] = extractelement <8 x i32> [[TMP50]], i32 7 ; DISABLED_MASKED_STRIDED-NEXT: [[TMP164:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP163]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP165:%.*]] = extractelement <8 x i8> [[TMP133]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP165]], i8* [[TMP164]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE60]] -; DISABLED_MASKED_STRIDED: pred.store.continue60: +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP162]], i8* [[TMP164]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE44]] +; DISABLED_MASKED_STRIDED: pred.store.continue44: ; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 ; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], -; DISABLED_MASKED_STRIDED-NEXT: [[TMP166:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP166]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP165:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP165]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; DISABLED_MASKED_STRIDED: for.end: ; DISABLED_MASKED_STRIDED-NEXT: ret void ; @@ -1508,328 +1475,304 @@ ; define dso_local void @masked_strided2_reverse(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i8 zeroext %guard) local_unnamed_addr { ; DISABLED_MASKED_STRIDED-LABEL: @masked_strided2_reverse( -; DISABLED_MASKED_STRIDED: br i1 {{.*}}, label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: entry: +; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32 +; DISABLED_MASKED_STRIDED-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, i8* [[Q:%.*]], i32 2048 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = getelementptr i8, i8* [[Q]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp ugt i8* [[TMP0]], [[SCEVGEP]] +; DISABLED_MASKED_STRIDED-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, i8* [[Q]], i32 2049 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr i8, i8* [[Q]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = icmp ugt i8* [[TMP2]], [[SCEVGEP2]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = or i1 [[TMP1]], [[TMP3]] +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP4]], label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]] ; DISABLED_MASKED_STRIDED: vector.ph: ; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i32 0 ; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer ; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; DISABLED_MASKED_STRIDED: vector.body: -; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE65:%.*]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ , [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE65]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], -; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = extractelement <8 x i1> [[TMP7]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP9]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE49:%.*]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ , [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE49]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], +; DISABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = extractelement <8 x i1> [[TMP5]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP8]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, i8* [[P:%.*]], i32 [[TMP10]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = load i8, i8* [[TMP11]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = insertelement <8 x i8> poison, i8 [[TMP12]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP6]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, i8* [[P:%.*]], i32 [[TMP8]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = load i8, i8* [[TMP9]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = insertelement <8 x i8> poison, i8 [[TMP10]], i32 0 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE]] ; DISABLED_MASKED_STRIDED: pred.load.continue: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP13]], [[PRED_LOAD_IF]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = extractelement <8 x i1> [[TMP7]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP11]], [[PRED_LOAD_IF]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = extractelement <8 x i1> [[TMP5]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP13]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if6: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <8 x i32> [[TMP8]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP16]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = load i8, i8* [[TMP17]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = insertelement <8 x i8> [[TMP14]], i8 [[TMP18]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[TMP6]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP14]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = load i8, i8* [[TMP15]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = insertelement <8 x i8> [[TMP12]], i8 [[TMP16]], i32 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE7]] ; DISABLED_MASKED_STRIDED: pred.load.continue7: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = phi <8 x i8> [ [[TMP14]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP19]], [[PRED_LOAD_IF6]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = extractelement <8 x i1> [[TMP7]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP21]], label [[PRED_LOAD_IF8:%.*]], label [[PRED_LOAD_CONTINUE9:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = phi <8 x i8> [ [[TMP12]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP17]], [[PRED_LOAD_IF6]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = extractelement <8 x i1> [[TMP5]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP19]], label [[PRED_LOAD_IF8:%.*]], label [[PRED_LOAD_CONTINUE9:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if8: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = extractelement <8 x i32> [[TMP8]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP22]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = load i8, i8* [[TMP23]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = insertelement <8 x i8> [[TMP20]], i8 [[TMP24]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = extractelement <8 x i32> [[TMP6]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP20]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = load i8, i8* [[TMP21]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = insertelement <8 x i8> [[TMP18]], i8 [[TMP22]], i32 2 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE9]] ; DISABLED_MASKED_STRIDED: pred.load.continue9: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = phi <8 x i8> [ [[TMP20]], [[PRED_LOAD_CONTINUE7]] ], [ [[TMP25]], [[PRED_LOAD_IF8]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = extractelement <8 x i1> [[TMP7]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP27]], label [[PRED_LOAD_IF10:%.*]], label [[PRED_LOAD_CONTINUE11:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = phi <8 x i8> [ [[TMP18]], [[PRED_LOAD_CONTINUE7]] ], [ [[TMP23]], [[PRED_LOAD_IF8]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = extractelement <8 x i1> [[TMP5]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP25]], label [[PRED_LOAD_IF10:%.*]], label [[PRED_LOAD_CONTINUE11:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if10: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = extractelement <8 x i32> [[TMP8]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP28]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = load i8, i8* [[TMP29]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = insertelement <8 x i8> [[TMP26]], i8 [[TMP30]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = extractelement <8 x i32> [[TMP6]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP26]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = load i8, i8* [[TMP27]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = insertelement <8 x i8> [[TMP24]], i8 [[TMP28]], i32 3 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE11]] ; DISABLED_MASKED_STRIDED: pred.load.continue11: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = phi <8 x i8> [ [[TMP26]], [[PRED_LOAD_CONTINUE9]] ], [ [[TMP31]], [[PRED_LOAD_IF10]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = extractelement <8 x i1> [[TMP7]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP33]], label [[PRED_LOAD_IF12:%.*]], label [[PRED_LOAD_CONTINUE13:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = phi <8 x i8> [ [[TMP24]], [[PRED_LOAD_CONTINUE9]] ], [ [[TMP29]], [[PRED_LOAD_IF10]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = extractelement <8 x i1> [[TMP5]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP31]], label [[PRED_LOAD_IF12:%.*]], label [[PRED_LOAD_CONTINUE13:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if12: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = extractelement <8 x i32> [[TMP8]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP34]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = load i8, i8* [[TMP35]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = insertelement <8 x i8> [[TMP32]], i8 [[TMP36]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = extractelement <8 x i32> [[TMP6]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP32]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = load i8, i8* [[TMP33]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = insertelement <8 x i8> [[TMP30]], i8 [[TMP34]], i32 4 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE13]] ; DISABLED_MASKED_STRIDED: pred.load.continue13: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = phi <8 x i8> [ [[TMP32]], [[PRED_LOAD_CONTINUE11]] ], [ [[TMP37]], [[PRED_LOAD_IF12]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = extractelement <8 x i1> [[TMP7]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP39]], label [[PRED_LOAD_IF14:%.*]], label [[PRED_LOAD_CONTINUE15:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = phi <8 x i8> [ [[TMP30]], [[PRED_LOAD_CONTINUE11]] ], [ [[TMP35]], [[PRED_LOAD_IF12]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = extractelement <8 x i1> [[TMP5]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP37]], label [[PRED_LOAD_IF14:%.*]], label [[PRED_LOAD_CONTINUE15:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if14: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = extractelement <8 x i32> [[TMP8]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP40]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = load i8, i8* [[TMP41]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = insertelement <8 x i8> [[TMP38]], i8 [[TMP42]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = extractelement <8 x i32> [[TMP6]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP38]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = load i8, i8* [[TMP39]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = insertelement <8 x i8> [[TMP36]], i8 [[TMP40]], i32 5 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE15]] ; DISABLED_MASKED_STRIDED: pred.load.continue15: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = phi <8 x i8> [ [[TMP38]], [[PRED_LOAD_CONTINUE13]] ], [ [[TMP43]], [[PRED_LOAD_IF14]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = extractelement <8 x i1> [[TMP7]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP45]], label [[PRED_LOAD_IF16:%.*]], label [[PRED_LOAD_CONTINUE17:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = phi <8 x i8> [ [[TMP36]], [[PRED_LOAD_CONTINUE13]] ], [ [[TMP41]], [[PRED_LOAD_IF14]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = extractelement <8 x i1> [[TMP5]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP43]], label [[PRED_LOAD_IF16:%.*]], label [[PRED_LOAD_CONTINUE17:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if16: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = extractelement <8 x i32> [[TMP8]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP46]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = load i8, i8* [[TMP47]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = insertelement <8 x i8> [[TMP44]], i8 [[TMP48]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = extractelement <8 x i32> [[TMP6]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP44]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = load i8, i8* [[TMP45]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = insertelement <8 x i8> [[TMP42]], i8 [[TMP46]], i32 6 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE17]] ; DISABLED_MASKED_STRIDED: pred.load.continue17: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = phi <8 x i8> [ [[TMP44]], [[PRED_LOAD_CONTINUE15]] ], [ [[TMP49]], [[PRED_LOAD_IF16]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = extractelement <8 x i1> [[TMP7]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label [[PRED_LOAD_IF18:%.*]], label [[PRED_LOAD_CONTINUE19:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = phi <8 x i8> [ [[TMP42]], [[PRED_LOAD_CONTINUE15]] ], [ [[TMP47]], [[PRED_LOAD_IF16]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = extractelement <8 x i1> [[TMP5]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP49]], label [[PRED_LOAD_IF18:%.*]], label [[PRED_LOAD_CONTINUE19:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if18: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = extractelement <8 x i32> [[TMP8]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP52]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP54:%.*]] = load i8, i8* [[TMP53]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP55:%.*]] = insertelement <8 x i8> [[TMP50]], i8 [[TMP54]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = extractelement <8 x i32> [[TMP6]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP50]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = load i8, i8* [[TMP51]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = insertelement <8 x i8> [[TMP48]], i8 [[TMP52]], i32 7 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE19]] ; DISABLED_MASKED_STRIDED: pred.load.continue19: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP56:%.*]] = phi <8 x i8> [ [[TMP50]], [[PRED_LOAD_CONTINUE17]] ], [ [[TMP55]], [[PRED_LOAD_IF18]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP57:%.*]] = or <8 x i32> [[TMP8]], -; DISABLED_MASKED_STRIDED-NEXT: [[TMP58:%.*]] = extractelement <8 x i1> [[TMP7]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP58]], label [[PRED_LOAD_IF20:%.*]], label [[PRED_LOAD_CONTINUE21:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP54:%.*]] = phi <8 x i8> [ [[TMP48]], [[PRED_LOAD_CONTINUE17]] ], [ [[TMP53]], [[PRED_LOAD_IF18]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP55:%.*]] = or <8 x i32> [[TMP6]], +; DISABLED_MASKED_STRIDED-NEXT: [[TMP56:%.*]] = extractelement <8 x i1> [[TMP5]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP56]], label [[PRED_LOAD_IF20:%.*]], label [[PRED_LOAD_CONTINUE21:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if20: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP59:%.*]] = extractelement <8 x i32> [[TMP57]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP60:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP59]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP61:%.*]] = load i8, i8* [[TMP60]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP62:%.*]] = insertelement <8 x i8> poison, i8 [[TMP61]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP57:%.*]] = extractelement <8 x i32> [[TMP55]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP58:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP57]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP59:%.*]] = load i8, i8* [[TMP58]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP60:%.*]] = insertelement <8 x i8> poison, i8 [[TMP59]], i32 0 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE21]] ; DISABLED_MASKED_STRIDED: pred.load.continue21: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP63:%.*]] = phi <8 x i8> [ poison, [[PRED_LOAD_CONTINUE19]] ], [ [[TMP62]], [[PRED_LOAD_IF20]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP64:%.*]] = extractelement <8 x i1> [[TMP7]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP64]], label [[PRED_LOAD_IF22:%.*]], label [[PRED_LOAD_CONTINUE23:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP61:%.*]] = phi <8 x i8> [ poison, [[PRED_LOAD_CONTINUE19]] ], [ [[TMP60]], [[PRED_LOAD_IF20]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP62:%.*]] = extractelement <8 x i1> [[TMP5]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP62]], label [[PRED_LOAD_IF22:%.*]], label [[PRED_LOAD_CONTINUE23:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if22: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP65:%.*]] = extractelement <8 x i32> [[TMP57]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP66:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP65]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP67:%.*]] = load i8, i8* [[TMP66]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP68:%.*]] = insertelement <8 x i8> [[TMP63]], i8 [[TMP67]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP63:%.*]] = extractelement <8 x i32> [[TMP55]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP64:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP63]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP65:%.*]] = load i8, i8* [[TMP64]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP66:%.*]] = insertelement <8 x i8> [[TMP61]], i8 [[TMP65]], i32 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE23]] ; DISABLED_MASKED_STRIDED: pred.load.continue23: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP69:%.*]] = phi <8 x i8> [ [[TMP63]], [[PRED_LOAD_CONTINUE21]] ], [ [[TMP68]], [[PRED_LOAD_IF22]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP70:%.*]] = extractelement <8 x i1> [[TMP7]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP70]], label [[PRED_LOAD_IF24:%.*]], label [[PRED_LOAD_CONTINUE25:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP67:%.*]] = phi <8 x i8> [ [[TMP61]], [[PRED_LOAD_CONTINUE21]] ], [ [[TMP66]], [[PRED_LOAD_IF22]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP68:%.*]] = extractelement <8 x i1> [[TMP5]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP68]], label [[PRED_LOAD_IF24:%.*]], label [[PRED_LOAD_CONTINUE25:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if24: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP71:%.*]] = extractelement <8 x i32> [[TMP57]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP72:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP71]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP73:%.*]] = load i8, i8* [[TMP72]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP74:%.*]] = insertelement <8 x i8> [[TMP69]], i8 [[TMP73]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP69:%.*]] = extractelement <8 x i32> [[TMP55]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP70:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP69]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP71:%.*]] = load i8, i8* [[TMP70]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP72:%.*]] = insertelement <8 x i8> [[TMP67]], i8 [[TMP71]], i32 2 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE25]] ; DISABLED_MASKED_STRIDED: pred.load.continue25: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP75:%.*]] = phi <8 x i8> [ [[TMP69]], [[PRED_LOAD_CONTINUE23]] ], [ [[TMP74]], [[PRED_LOAD_IF24]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP76:%.*]] = extractelement <8 x i1> [[TMP7]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP76]], label [[PRED_LOAD_IF26:%.*]], label [[PRED_LOAD_CONTINUE27:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP73:%.*]] = phi <8 x i8> [ [[TMP67]], [[PRED_LOAD_CONTINUE23]] ], [ [[TMP72]], [[PRED_LOAD_IF24]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP74:%.*]] = extractelement <8 x i1> [[TMP5]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP74]], label [[PRED_LOAD_IF26:%.*]], label [[PRED_LOAD_CONTINUE27:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if26: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP77:%.*]] = extractelement <8 x i32> [[TMP57]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP78:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP77]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP79:%.*]] = load i8, i8* [[TMP78]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP80:%.*]] = insertelement <8 x i8> [[TMP75]], i8 [[TMP79]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP75:%.*]] = extractelement <8 x i32> [[TMP55]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP76:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP75]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP77:%.*]] = load i8, i8* [[TMP76]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP78:%.*]] = insertelement <8 x i8> [[TMP73]], i8 [[TMP77]], i32 3 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE27]] ; DISABLED_MASKED_STRIDED: pred.load.continue27: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP81:%.*]] = phi <8 x i8> [ [[TMP75]], [[PRED_LOAD_CONTINUE25]] ], [ [[TMP80]], [[PRED_LOAD_IF26]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP82:%.*]] = extractelement <8 x i1> [[TMP7]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP82]], label [[PRED_LOAD_IF28:%.*]], label [[PRED_LOAD_CONTINUE29:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP79:%.*]] = phi <8 x i8> [ [[TMP73]], [[PRED_LOAD_CONTINUE25]] ], [ [[TMP78]], [[PRED_LOAD_IF26]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP80:%.*]] = extractelement <8 x i1> [[TMP5]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP80]], label [[PRED_LOAD_IF28:%.*]], label [[PRED_LOAD_CONTINUE29:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if28: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP83:%.*]] = extractelement <8 x i32> [[TMP57]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP84:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP83]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP85:%.*]] = load i8, i8* [[TMP84]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP86:%.*]] = insertelement <8 x i8> [[TMP81]], i8 [[TMP85]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP81:%.*]] = extractelement <8 x i32> [[TMP55]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP82:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP81]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP83:%.*]] = load i8, i8* [[TMP82]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP84:%.*]] = insertelement <8 x i8> [[TMP79]], i8 [[TMP83]], i32 4 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE29]] ; DISABLED_MASKED_STRIDED: pred.load.continue29: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP87:%.*]] = phi <8 x i8> [ [[TMP81]], [[PRED_LOAD_CONTINUE27]] ], [ [[TMP86]], [[PRED_LOAD_IF28]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP88:%.*]] = extractelement <8 x i1> [[TMP7]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP88]], label [[PRED_LOAD_IF30:%.*]], label [[PRED_LOAD_CONTINUE31:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP85:%.*]] = phi <8 x i8> [ [[TMP79]], [[PRED_LOAD_CONTINUE27]] ], [ [[TMP84]], [[PRED_LOAD_IF28]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP86:%.*]] = extractelement <8 x i1> [[TMP5]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP86]], label [[PRED_LOAD_IF30:%.*]], label [[PRED_LOAD_CONTINUE31:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if30: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP89:%.*]] = extractelement <8 x i32> [[TMP57]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP90:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP89]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP91:%.*]] = load i8, i8* [[TMP90]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP92:%.*]] = insertelement <8 x i8> [[TMP87]], i8 [[TMP91]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP87:%.*]] = extractelement <8 x i32> [[TMP55]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP88:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP87]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP89:%.*]] = load i8, i8* [[TMP88]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP90:%.*]] = insertelement <8 x i8> [[TMP85]], i8 [[TMP89]], i32 5 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE31]] ; DISABLED_MASKED_STRIDED: pred.load.continue31: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP93:%.*]] = phi <8 x i8> [ [[TMP87]], [[PRED_LOAD_CONTINUE29]] ], [ [[TMP92]], [[PRED_LOAD_IF30]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP94:%.*]] = extractelement <8 x i1> [[TMP7]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP94]], label [[PRED_LOAD_IF32:%.*]], label [[PRED_LOAD_CONTINUE33:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP91:%.*]] = phi <8 x i8> [ [[TMP85]], [[PRED_LOAD_CONTINUE29]] ], [ [[TMP90]], [[PRED_LOAD_IF30]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP92:%.*]] = extractelement <8 x i1> [[TMP5]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP92]], label [[PRED_LOAD_IF32:%.*]], label [[PRED_LOAD_CONTINUE33:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if32: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP95:%.*]] = extractelement <8 x i32> [[TMP57]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP96:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP95]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP97:%.*]] = load i8, i8* [[TMP96]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP98:%.*]] = insertelement <8 x i8> [[TMP93]], i8 [[TMP97]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP93:%.*]] = extractelement <8 x i32> [[TMP55]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP94:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP93]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP95:%.*]] = load i8, i8* [[TMP94]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP96:%.*]] = insertelement <8 x i8> [[TMP91]], i8 [[TMP95]], i32 6 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE33]] ; DISABLED_MASKED_STRIDED: pred.load.continue33: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP99:%.*]] = phi <8 x i8> [ [[TMP93]], [[PRED_LOAD_CONTINUE31]] ], [ [[TMP98]], [[PRED_LOAD_IF32]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP100:%.*]] = extractelement <8 x i1> [[TMP7]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label [[PRED_LOAD_IF34:%.*]], label [[PRED_LOAD_CONTINUE35:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP97:%.*]] = phi <8 x i8> [ [[TMP91]], [[PRED_LOAD_CONTINUE31]] ], [ [[TMP96]], [[PRED_LOAD_IF32]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP98:%.*]] = extractelement <8 x i1> [[TMP5]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP98]], label [[PRED_LOAD_IF34:%.*]], label [[PRED_LOAD_CONTINUE35:%.*]] ; DISABLED_MASKED_STRIDED: pred.load.if34: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP101:%.*]] = extractelement <8 x i32> [[TMP57]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP102:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP101]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP103:%.*]] = load i8, i8* [[TMP102]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP104:%.*]] = insertelement <8 x i8> [[TMP99]], i8 [[TMP103]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP99:%.*]] = extractelement <8 x i32> [[TMP55]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP100:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP99]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP101:%.*]] = load i8, i8* [[TMP100]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP102:%.*]] = insertelement <8 x i8> [[TMP97]], i8 [[TMP101]], i32 7 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE35]] ; DISABLED_MASKED_STRIDED: pred.load.continue35: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP105:%.*]] = phi <8 x i8> [ [[TMP99]], [[PRED_LOAD_CONTINUE33]] ], [ [[TMP104]], [[PRED_LOAD_IF34]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = icmp slt <8 x i8> [[TMP56]], [[TMP105]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = select <8 x i1> [[TMP106]], <8 x i8> [[TMP105]], <8 x i8> [[TMP56]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = extractelement <8 x i1> [[TMP7]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP103:%.*]] = phi <8 x i8> [ [[TMP97]], [[PRED_LOAD_CONTINUE33]] ], [ [[TMP102]], [[PRED_LOAD_IF34]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP104:%.*]] = icmp slt <8 x i8> [[TMP54]], [[TMP103]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP105:%.*]] = select <8 x i1> [[TMP104]], <8 x i8> [[TMP103]], <8 x i8> [[TMP54]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = extractelement <8 x i1> [[TMP5]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP106]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = extractelement <8 x i32> [[TMP8]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP109]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = extractelement <8 x i8> [[TMP107]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP111]], i8* [[TMP110]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = extractelement <8 x i32> [[TMP6]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP107]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = extractelement <8 x i8> [[TMP105]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP109]], i8* [[TMP108]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = extractelement <8 x i8> [[TMP105]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = sub i8 0, [[TMP110]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = extractelement <8 x i32> [[TMP55]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP112]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP111]], i8* [[TMP113]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE]] ; DISABLED_MASKED_STRIDED: pred.store.continue: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = extractelement <8 x i1> [[TMP7]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label [[PRED_STORE_IF36:%.*]], label [[PRED_STORE_CONTINUE37:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = extractelement <8 x i1> [[TMP5]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP114]], label [[PRED_STORE_IF36:%.*]], label [[PRED_STORE_CONTINUE37:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if36: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = extractelement <8 x i32> [[TMP8]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP113]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = extractelement <8 x i8> [[TMP107]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP115]], i8* [[TMP114]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = extractelement <8 x i32> [[TMP6]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP115]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = extractelement <8 x i8> [[TMP105]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP117]], i8* [[TMP116]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = extractelement <8 x i8> [[TMP105]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = sub i8 0, [[TMP118]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = extractelement <8 x i32> [[TMP55]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP120]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP119]], i8* [[TMP121]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE37]] ; DISABLED_MASKED_STRIDED: pred.store.continue37: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = extractelement <8 x i1> [[TMP7]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label [[PRED_STORE_IF38:%.*]], label [[PRED_STORE_CONTINUE39:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = extractelement <8 x i1> [[TMP5]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP122]], label [[PRED_STORE_IF38:%.*]], label [[PRED_STORE_CONTINUE39:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if38: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = extractelement <8 x i32> [[TMP8]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP117]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = extractelement <8 x i8> [[TMP107]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP119]], i8* [[TMP118]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = extractelement <8 x i32> [[TMP6]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP123]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = extractelement <8 x i8> [[TMP105]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP125]], i8* [[TMP124]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = extractelement <8 x i8> [[TMP105]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = sub i8 0, [[TMP126]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = extractelement <8 x i32> [[TMP55]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP128]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP127]], i8* [[TMP129]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE39]] ; DISABLED_MASKED_STRIDED: pred.store.continue39: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = extractelement <8 x i1> [[TMP7]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label [[PRED_STORE_IF40:%.*]], label [[PRED_STORE_CONTINUE41:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = extractelement <8 x i1> [[TMP5]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP130]], label [[PRED_STORE_IF40:%.*]], label [[PRED_STORE_CONTINUE41:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if40: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = extractelement <8 x i32> [[TMP8]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP121]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = extractelement <8 x i8> [[TMP107]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP123]], i8* [[TMP122]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = extractelement <8 x i32> [[TMP6]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP131]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = extractelement <8 x i8> [[TMP105]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP133]], i8* [[TMP132]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = extractelement <8 x i8> [[TMP105]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = sub i8 0, [[TMP134]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i32> [[TMP55]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP136]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP135]], i8* [[TMP137]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE41]] ; DISABLED_MASKED_STRIDED: pred.store.continue41: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = extractelement <8 x i1> [[TMP7]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label [[PRED_STORE_IF42:%.*]], label [[PRED_STORE_CONTINUE43:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i1> [[TMP5]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP138]], label [[PRED_STORE_IF42:%.*]], label [[PRED_STORE_CONTINUE43:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if42: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = extractelement <8 x i32> [[TMP8]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP125]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = extractelement <8 x i8> [[TMP107]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP127]], i8* [[TMP126]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = extractelement <8 x i32> [[TMP6]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP139]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i8> [[TMP105]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP141]], i8* [[TMP140]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i8> [[TMP105]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = sub i8 0, [[TMP142]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i32> [[TMP55]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP144]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP143]], i8* [[TMP145]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE43]] ; DISABLED_MASKED_STRIDED: pred.store.continue43: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = extractelement <8 x i1> [[TMP7]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label [[PRED_STORE_IF44:%.*]], label [[PRED_STORE_CONTINUE45:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = extractelement <8 x i1> [[TMP5]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP146]], label [[PRED_STORE_IF44:%.*]], label [[PRED_STORE_CONTINUE45:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if44: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = extractelement <8 x i32> [[TMP8]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP129]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = extractelement <8 x i8> [[TMP107]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP131]], i8* [[TMP130]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = extractelement <8 x i32> [[TMP6]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP147]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = extractelement <8 x i8> [[TMP105]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP149]], i8* [[TMP148]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i8> [[TMP105]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = sub i8 0, [[TMP150]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = extractelement <8 x i32> [[TMP55]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP152]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP151]], i8* [[TMP153]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE45]] ; DISABLED_MASKED_STRIDED: pred.store.continue45: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = extractelement <8 x i1> [[TMP7]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP132]], label [[PRED_STORE_IF46:%.*]], label [[PRED_STORE_CONTINUE47:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i1> [[TMP5]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP154]], label [[PRED_STORE_IF46:%.*]], label [[PRED_STORE_CONTINUE47:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if46: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = extractelement <8 x i32> [[TMP8]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP133]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = extractelement <8 x i8> [[TMP107]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP135]], i8* [[TMP134]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = extractelement <8 x i32> [[TMP6]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP155]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = extractelement <8 x i8> [[TMP105]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP157]], i8* [[TMP156]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = extractelement <8 x i8> [[TMP105]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = sub i8 0, [[TMP158]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = extractelement <8 x i32> [[TMP55]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP161:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP160]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP159]], i8* [[TMP161]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE47]] ; DISABLED_MASKED_STRIDED: pred.store.continue47: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i1> [[TMP7]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP136]], label [[PRED_STORE_IF48:%.*]], label [[PRED_STORE_CONTINUE49:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = extractelement <8 x i1> [[TMP5]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP162]], label [[PRED_STORE_IF48:%.*]], label [[PRED_STORE_CONTINUE49]] ; DISABLED_MASKED_STRIDED: pred.store.if48: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = extractelement <8 x i32> [[TMP8]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP137]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = extractelement <8 x i8> [[TMP107]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP139]], i8* [[TMP138]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP163:%.*]] = extractelement <8 x i32> [[TMP6]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP164:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP163]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP165:%.*]] = extractelement <8 x i8> [[TMP105]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP165]], i8* [[TMP164]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP166:%.*]] = extractelement <8 x i8> [[TMP105]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP167:%.*]] = sub i8 0, [[TMP166]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP168:%.*]] = extractelement <8 x i32> [[TMP55]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP169:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP168]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP167]], i8* [[TMP169]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE49]] ; DISABLED_MASKED_STRIDED: pred.store.continue49: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = sub <8 x i8> zeroinitializer, [[TMP107]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i1> [[TMP7]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP141]], label [[PRED_STORE_IF50:%.*]], label [[PRED_STORE_CONTINUE51:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if50: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i32> [[TMP57]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP142]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i8> [[TMP140]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP144]], i8* [[TMP143]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE51]] -; DISABLED_MASKED_STRIDED: pred.store.continue51: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = extractelement <8 x i1> [[TMP7]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP145]], label [[PRED_STORE_IF52:%.*]], label [[PRED_STORE_CONTINUE53:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if52: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = extractelement <8 x i32> [[TMP57]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP146]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = extractelement <8 x i8> [[TMP140]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP148]], i8* [[TMP147]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE53]] -; DISABLED_MASKED_STRIDED: pred.store.continue53: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = extractelement <8 x i1> [[TMP7]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP149]], label [[PRED_STORE_IF54:%.*]], label [[PRED_STORE_CONTINUE55:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if54: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i32> [[TMP57]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP150]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = extractelement <8 x i8> [[TMP140]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP152]], i8* [[TMP151]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE55]] -; DISABLED_MASKED_STRIDED: pred.store.continue55: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = extractelement <8 x i1> [[TMP7]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP153]], label [[PRED_STORE_IF56:%.*]], label [[PRED_STORE_CONTINUE57:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if56: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i32> [[TMP57]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP154]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = extractelement <8 x i8> [[TMP140]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP156]], i8* [[TMP155]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE57]] -; DISABLED_MASKED_STRIDED: pred.store.continue57: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = extractelement <8 x i1> [[TMP7]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP157]], label [[PRED_STORE_IF58:%.*]], label [[PRED_STORE_CONTINUE59:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if58: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = extractelement <8 x i32> [[TMP57]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP158]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = extractelement <8 x i8> [[TMP140]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP160]], i8* [[TMP159]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE59]] -; DISABLED_MASKED_STRIDED: pred.store.continue59: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP161:%.*]] = extractelement <8 x i1> [[TMP7]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP161]], label [[PRED_STORE_IF60:%.*]], label [[PRED_STORE_CONTINUE61:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if60: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = extractelement <8 x i32> [[TMP57]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP163:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP162]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP164:%.*]] = extractelement <8 x i8> [[TMP140]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP164]], i8* [[TMP163]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE61]] -; DISABLED_MASKED_STRIDED: pred.store.continue61: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP165:%.*]] = extractelement <8 x i1> [[TMP7]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP165]], label [[PRED_STORE_IF62:%.*]], label [[PRED_STORE_CONTINUE63:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if62: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP166:%.*]] = extractelement <8 x i32> [[TMP57]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP167:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP166]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP168:%.*]] = extractelement <8 x i8> [[TMP140]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP168]], i8* [[TMP167]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE63]] -; DISABLED_MASKED_STRIDED: pred.store.continue63: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP169:%.*]] = extractelement <8 x i1> [[TMP7]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP169]], label [[PRED_STORE_IF64:%.*]], label [[PRED_STORE_CONTINUE65]] -; DISABLED_MASKED_STRIDED: pred.store.if64: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP170:%.*]] = extractelement <8 x i32> [[TMP57]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP171:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP170]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP172:%.*]] = extractelement <8 x i8> [[TMP140]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP172]], i8* [[TMP171]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE65]] -; DISABLED_MASKED_STRIDED: pred.store.continue65: ; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 ; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], -; DISABLED_MASKED_STRIDED-NEXT: [[TMP173:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP173]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP170:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP170]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; DISABLED_MASKED_STRIDED: for.body: ; DISABLED_MASKED_STRIDED-NEXT: [[IX_024:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_INC:%.*]] ], [ 1024, [[ENTRY:%.*]] ] ; DISABLED_MASKED_STRIDED-NEXT: [[CMP1:%.*]] = icmp ugt i32 [[IX_024]], [[CONV]] @@ -1837,12 +1780,12 @@ ; DISABLED_MASKED_STRIDED: if.then: ; DISABLED_MASKED_STRIDED-NEXT: [[MUL:%.*]] = shl nuw nsw i32 [[IX_024]], 1 ; DISABLED_MASKED_STRIDED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[MUL]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP174:%.*]] = load i8, i8* [[ARRAYIDX]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP171:%.*]] = load i8, i8* [[ARRAYIDX]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: [[ADD:%.*]] = or i32 [[MUL]], 1 ; DISABLED_MASKED_STRIDED-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[ADD]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP175:%.*]] = load i8, i8* [[ARRAYIDX4]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: [[CMP_I:%.*]] = icmp slt i8 [[TMP174]], [[TMP175]] -; DISABLED_MASKED_STRIDED-NEXT: [[SPEC_SELECT_I:%.*]] = select i1 [[CMP_I]], i8 [[TMP175]], i8 [[TMP174]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP172:%.*]] = load i8, i8* [[ARRAYIDX4]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[CMP_I:%.*]] = icmp slt i8 [[TMP171]], [[TMP172]] +; DISABLED_MASKED_STRIDED-NEXT: [[SPEC_SELECT_I:%.*]] = select i1 [[CMP_I]], i8 [[TMP172]], i8 [[TMP171]] ; DISABLED_MASKED_STRIDED-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[MUL]] ; DISABLED_MASKED_STRIDED-NEXT: store i8 [[SPEC_SELECT_I]], i8* [[ARRAYIDX6]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: [[SUB:%.*]] = sub i8 0, [[SPEC_SELECT_I]] @@ -1852,333 +1795,309 @@ ; DISABLED_MASKED_STRIDED: for.inc: ; DISABLED_MASKED_STRIDED-NEXT: [[INC]] = add nsw i32 [[IX_024]], -1 ; DISABLED_MASKED_STRIDED-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 0 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]] +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] ; DISABLED_MASKED_STRIDED: for.end: ; DISABLED_MASKED_STRIDED-NEXT: ret void ; ; ENABLED_MASKED_STRIDED-LABEL: @masked_strided2_reverse( -; ENABLED_MASKED_STRIDED: br i1 {{%.*}}, label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: entry: +; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32 +; ENABLED_MASKED_STRIDED-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, i8* [[Q:%.*]], i32 2048 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = getelementptr i8, i8* [[Q]], i32 2 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp ugt i8* [[TMP0]], [[SCEVGEP]] +; ENABLED_MASKED_STRIDED-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, i8* [[Q]], i32 2049 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr i8, i8* [[Q]], i32 3 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = icmp ugt i8* [[TMP2]], [[SCEVGEP2]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = or i1 [[TMP1]], [[TMP3]] +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP4]], label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]] ; ENABLED_MASKED_STRIDED: vector.ph: ; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i32 0 ; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer ; ENABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; ENABLED_MASKED_STRIDED: vector.body: -; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE65:%.*]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ , [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE65]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], -; ENABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = extractelement <8 x i1> [[TMP7]], i32 0 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP9]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE49:%.*]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ , [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE49]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], +; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = extractelement <8 x i1> [[TMP5]], i32 0 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP8]], i32 0 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, i8* [[P:%.*]], i32 [[TMP10]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = load i8, i8* [[TMP11]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = insertelement <8 x i8> poison, i8 [[TMP12]], i32 0 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP6]], i32 0 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, i8* [[P:%.*]], i32 [[TMP8]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = load i8, i8* [[TMP9]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = insertelement <8 x i8> poison, i8 [[TMP10]], i32 0 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE]] ; ENABLED_MASKED_STRIDED: pred.load.continue: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP13]], [[PRED_LOAD_IF]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = extractelement <8 x i1> [[TMP7]], i32 1 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = phi <8 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP11]], [[PRED_LOAD_IF]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = extractelement <8 x i1> [[TMP5]], i32 1 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP13]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if6: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <8 x i32> [[TMP8]], i32 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP16]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = load i8, i8* [[TMP17]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = insertelement <8 x i8> [[TMP14]], i8 [[TMP18]], i32 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[TMP6]], i32 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP14]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = load i8, i8* [[TMP15]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = insertelement <8 x i8> [[TMP12]], i8 [[TMP16]], i32 1 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE7]] ; ENABLED_MASKED_STRIDED: pred.load.continue7: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = phi <8 x i8> [ [[TMP14]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP19]], [[PRED_LOAD_IF6]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = extractelement <8 x i1> [[TMP7]], i32 2 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP21]], label [[PRED_LOAD_IF8:%.*]], label [[PRED_LOAD_CONTINUE9:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = phi <8 x i8> [ [[TMP12]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP17]], [[PRED_LOAD_IF6]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP19:%.*]] = extractelement <8 x i1> [[TMP5]], i32 2 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP19]], label [[PRED_LOAD_IF8:%.*]], label [[PRED_LOAD_CONTINUE9:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if8: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = extractelement <8 x i32> [[TMP8]], i32 2 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP22]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = load i8, i8* [[TMP23]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = insertelement <8 x i8> [[TMP20]], i8 [[TMP24]], i32 2 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP20:%.*]] = extractelement <8 x i32> [[TMP6]], i32 2 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP20]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = load i8, i8* [[TMP21]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = insertelement <8 x i8> [[TMP18]], i8 [[TMP22]], i32 2 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE9]] ; ENABLED_MASKED_STRIDED: pred.load.continue9: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = phi <8 x i8> [ [[TMP20]], [[PRED_LOAD_CONTINUE7]] ], [ [[TMP25]], [[PRED_LOAD_IF8]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = extractelement <8 x i1> [[TMP7]], i32 3 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP27]], label [[PRED_LOAD_IF10:%.*]], label [[PRED_LOAD_CONTINUE11:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = phi <8 x i8> [ [[TMP18]], [[PRED_LOAD_CONTINUE7]] ], [ [[TMP23]], [[PRED_LOAD_IF8]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = extractelement <8 x i1> [[TMP5]], i32 3 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP25]], label [[PRED_LOAD_IF10:%.*]], label [[PRED_LOAD_CONTINUE11:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if10: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = extractelement <8 x i32> [[TMP8]], i32 3 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP28]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = load i8, i8* [[TMP29]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = insertelement <8 x i8> [[TMP26]], i8 [[TMP30]], i32 3 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = extractelement <8 x i32> [[TMP6]], i32 3 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP26]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = load i8, i8* [[TMP27]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = insertelement <8 x i8> [[TMP24]], i8 [[TMP28]], i32 3 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE11]] ; ENABLED_MASKED_STRIDED: pred.load.continue11: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = phi <8 x i8> [ [[TMP26]], [[PRED_LOAD_CONTINUE9]] ], [ [[TMP31]], [[PRED_LOAD_IF10]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = extractelement <8 x i1> [[TMP7]], i32 4 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP33]], label [[PRED_LOAD_IF12:%.*]], label [[PRED_LOAD_CONTINUE13:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = phi <8 x i8> [ [[TMP24]], [[PRED_LOAD_CONTINUE9]] ], [ [[TMP29]], [[PRED_LOAD_IF10]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = extractelement <8 x i1> [[TMP5]], i32 4 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP31]], label [[PRED_LOAD_IF12:%.*]], label [[PRED_LOAD_CONTINUE13:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if12: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = extractelement <8 x i32> [[TMP8]], i32 4 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP34]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = load i8, i8* [[TMP35]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = insertelement <8 x i8> [[TMP32]], i8 [[TMP36]], i32 4 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = extractelement <8 x i32> [[TMP6]], i32 4 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP32]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP34:%.*]] = load i8, i8* [[TMP33]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP35:%.*]] = insertelement <8 x i8> [[TMP30]], i8 [[TMP34]], i32 4 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE13]] ; ENABLED_MASKED_STRIDED: pred.load.continue13: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = phi <8 x i8> [ [[TMP32]], [[PRED_LOAD_CONTINUE11]] ], [ [[TMP37]], [[PRED_LOAD_IF12]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = extractelement <8 x i1> [[TMP7]], i32 5 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP39]], label [[PRED_LOAD_IF14:%.*]], label [[PRED_LOAD_CONTINUE15:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP36:%.*]] = phi <8 x i8> [ [[TMP30]], [[PRED_LOAD_CONTINUE11]] ], [ [[TMP35]], [[PRED_LOAD_IF12]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP37:%.*]] = extractelement <8 x i1> [[TMP5]], i32 5 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP37]], label [[PRED_LOAD_IF14:%.*]], label [[PRED_LOAD_CONTINUE15:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if14: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = extractelement <8 x i32> [[TMP8]], i32 5 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP40]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = load i8, i8* [[TMP41]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = insertelement <8 x i8> [[TMP38]], i8 [[TMP42]], i32 5 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP38:%.*]] = extractelement <8 x i32> [[TMP6]], i32 5 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP39:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP38]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP40:%.*]] = load i8, i8* [[TMP39]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP41:%.*]] = insertelement <8 x i8> [[TMP36]], i8 [[TMP40]], i32 5 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE15]] ; ENABLED_MASKED_STRIDED: pred.load.continue15: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = phi <8 x i8> [ [[TMP38]], [[PRED_LOAD_CONTINUE13]] ], [ [[TMP43]], [[PRED_LOAD_IF14]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = extractelement <8 x i1> [[TMP7]], i32 6 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP45]], label [[PRED_LOAD_IF16:%.*]], label [[PRED_LOAD_CONTINUE17:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP42:%.*]] = phi <8 x i8> [ [[TMP36]], [[PRED_LOAD_CONTINUE13]] ], [ [[TMP41]], [[PRED_LOAD_IF14]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP43:%.*]] = extractelement <8 x i1> [[TMP5]], i32 6 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP43]], label [[PRED_LOAD_IF16:%.*]], label [[PRED_LOAD_CONTINUE17:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if16: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = extractelement <8 x i32> [[TMP8]], i32 6 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP46]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = load i8, i8* [[TMP47]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = insertelement <8 x i8> [[TMP44]], i8 [[TMP48]], i32 6 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP44:%.*]] = extractelement <8 x i32> [[TMP6]], i32 6 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP45:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP44]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP46:%.*]] = load i8, i8* [[TMP45]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP47:%.*]] = insertelement <8 x i8> [[TMP42]], i8 [[TMP46]], i32 6 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE17]] ; ENABLED_MASKED_STRIDED: pred.load.continue17: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = phi <8 x i8> [ [[TMP44]], [[PRED_LOAD_CONTINUE15]] ], [ [[TMP49]], [[PRED_LOAD_IF16]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = extractelement <8 x i1> [[TMP7]], i32 7 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label [[PRED_LOAD_IF18:%.*]], label [[PRED_LOAD_CONTINUE19:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP48:%.*]] = phi <8 x i8> [ [[TMP42]], [[PRED_LOAD_CONTINUE15]] ], [ [[TMP47]], [[PRED_LOAD_IF16]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = extractelement <8 x i1> [[TMP5]], i32 7 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP49]], label [[PRED_LOAD_IF18:%.*]], label [[PRED_LOAD_CONTINUE19:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if18: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = extractelement <8 x i32> [[TMP8]], i32 7 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP52]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP54:%.*]] = load i8, i8* [[TMP53]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP55:%.*]] = insertelement <8 x i8> [[TMP50]], i8 [[TMP54]], i32 7 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = extractelement <8 x i32> [[TMP6]], i32 7 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP50]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = load i8, i8* [[TMP51]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = insertelement <8 x i8> [[TMP48]], i8 [[TMP52]], i32 7 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE19]] ; ENABLED_MASKED_STRIDED: pred.load.continue19: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP56:%.*]] = phi <8 x i8> [ [[TMP50]], [[PRED_LOAD_CONTINUE17]] ], [ [[TMP55]], [[PRED_LOAD_IF18]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP57:%.*]] = or <8 x i32> [[TMP8]], -; ENABLED_MASKED_STRIDED-NEXT: [[TMP58:%.*]] = extractelement <8 x i1> [[TMP7]], i32 0 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP58]], label [[PRED_LOAD_IF20:%.*]], label [[PRED_LOAD_CONTINUE21:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP54:%.*]] = phi <8 x i8> [ [[TMP48]], [[PRED_LOAD_CONTINUE17]] ], [ [[TMP53]], [[PRED_LOAD_IF18]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP55:%.*]] = or <8 x i32> [[TMP6]], +; ENABLED_MASKED_STRIDED-NEXT: [[TMP56:%.*]] = extractelement <8 x i1> [[TMP5]], i32 0 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP56]], label [[PRED_LOAD_IF20:%.*]], label [[PRED_LOAD_CONTINUE21:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if20: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP59:%.*]] = extractelement <8 x i32> [[TMP57]], i32 0 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP60:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP59]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP61:%.*]] = load i8, i8* [[TMP60]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP62:%.*]] = insertelement <8 x i8> poison, i8 [[TMP61]], i32 0 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP57:%.*]] = extractelement <8 x i32> [[TMP55]], i32 0 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP58:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP57]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP59:%.*]] = load i8, i8* [[TMP58]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP60:%.*]] = insertelement <8 x i8> poison, i8 [[TMP59]], i32 0 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE21]] ; ENABLED_MASKED_STRIDED: pred.load.continue21: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP63:%.*]] = phi <8 x i8> [ poison, [[PRED_LOAD_CONTINUE19]] ], [ [[TMP62]], [[PRED_LOAD_IF20]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP64:%.*]] = extractelement <8 x i1> [[TMP7]], i32 1 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP64]], label [[PRED_LOAD_IF22:%.*]], label [[PRED_LOAD_CONTINUE23:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP61:%.*]] = phi <8 x i8> [ poison, [[PRED_LOAD_CONTINUE19]] ], [ [[TMP60]], [[PRED_LOAD_IF20]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP62:%.*]] = extractelement <8 x i1> [[TMP5]], i32 1 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP62]], label [[PRED_LOAD_IF22:%.*]], label [[PRED_LOAD_CONTINUE23:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if22: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP65:%.*]] = extractelement <8 x i32> [[TMP57]], i32 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP66:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP65]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP67:%.*]] = load i8, i8* [[TMP66]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP68:%.*]] = insertelement <8 x i8> [[TMP63]], i8 [[TMP67]], i32 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP63:%.*]] = extractelement <8 x i32> [[TMP55]], i32 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP64:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP63]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP65:%.*]] = load i8, i8* [[TMP64]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP66:%.*]] = insertelement <8 x i8> [[TMP61]], i8 [[TMP65]], i32 1 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE23]] ; ENABLED_MASKED_STRIDED: pred.load.continue23: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP69:%.*]] = phi <8 x i8> [ [[TMP63]], [[PRED_LOAD_CONTINUE21]] ], [ [[TMP68]], [[PRED_LOAD_IF22]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP70:%.*]] = extractelement <8 x i1> [[TMP7]], i32 2 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP70]], label [[PRED_LOAD_IF24:%.*]], label [[PRED_LOAD_CONTINUE25:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP67:%.*]] = phi <8 x i8> [ [[TMP61]], [[PRED_LOAD_CONTINUE21]] ], [ [[TMP66]], [[PRED_LOAD_IF22]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP68:%.*]] = extractelement <8 x i1> [[TMP5]], i32 2 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP68]], label [[PRED_LOAD_IF24:%.*]], label [[PRED_LOAD_CONTINUE25:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if24: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP71:%.*]] = extractelement <8 x i32> [[TMP57]], i32 2 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP72:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP71]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP73:%.*]] = load i8, i8* [[TMP72]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP74:%.*]] = insertelement <8 x i8> [[TMP69]], i8 [[TMP73]], i32 2 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP69:%.*]] = extractelement <8 x i32> [[TMP55]], i32 2 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP70:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP69]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP71:%.*]] = load i8, i8* [[TMP70]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP72:%.*]] = insertelement <8 x i8> [[TMP67]], i8 [[TMP71]], i32 2 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE25]] ; ENABLED_MASKED_STRIDED: pred.load.continue25: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP75:%.*]] = phi <8 x i8> [ [[TMP69]], [[PRED_LOAD_CONTINUE23]] ], [ [[TMP74]], [[PRED_LOAD_IF24]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP76:%.*]] = extractelement <8 x i1> [[TMP7]], i32 3 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP76]], label [[PRED_LOAD_IF26:%.*]], label [[PRED_LOAD_CONTINUE27:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP73:%.*]] = phi <8 x i8> [ [[TMP67]], [[PRED_LOAD_CONTINUE23]] ], [ [[TMP72]], [[PRED_LOAD_IF24]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP74:%.*]] = extractelement <8 x i1> [[TMP5]], i32 3 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP74]], label [[PRED_LOAD_IF26:%.*]], label [[PRED_LOAD_CONTINUE27:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if26: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP77:%.*]] = extractelement <8 x i32> [[TMP57]], i32 3 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP78:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP77]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP79:%.*]] = load i8, i8* [[TMP78]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP80:%.*]] = insertelement <8 x i8> [[TMP75]], i8 [[TMP79]], i32 3 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP75:%.*]] = extractelement <8 x i32> [[TMP55]], i32 3 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP76:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP75]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP77:%.*]] = load i8, i8* [[TMP76]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP78:%.*]] = insertelement <8 x i8> [[TMP73]], i8 [[TMP77]], i32 3 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE27]] ; ENABLED_MASKED_STRIDED: pred.load.continue27: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP81:%.*]] = phi <8 x i8> [ [[TMP75]], [[PRED_LOAD_CONTINUE25]] ], [ [[TMP80]], [[PRED_LOAD_IF26]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP82:%.*]] = extractelement <8 x i1> [[TMP7]], i32 4 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP82]], label [[PRED_LOAD_IF28:%.*]], label [[PRED_LOAD_CONTINUE29:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP79:%.*]] = phi <8 x i8> [ [[TMP73]], [[PRED_LOAD_CONTINUE25]] ], [ [[TMP78]], [[PRED_LOAD_IF26]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP80:%.*]] = extractelement <8 x i1> [[TMP5]], i32 4 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP80]], label [[PRED_LOAD_IF28:%.*]], label [[PRED_LOAD_CONTINUE29:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if28: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP83:%.*]] = extractelement <8 x i32> [[TMP57]], i32 4 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP84:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP83]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP85:%.*]] = load i8, i8* [[TMP84]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP86:%.*]] = insertelement <8 x i8> [[TMP81]], i8 [[TMP85]], i32 4 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP81:%.*]] = extractelement <8 x i32> [[TMP55]], i32 4 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP82:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP81]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP83:%.*]] = load i8, i8* [[TMP82]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP84:%.*]] = insertelement <8 x i8> [[TMP79]], i8 [[TMP83]], i32 4 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE29]] ; ENABLED_MASKED_STRIDED: pred.load.continue29: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP87:%.*]] = phi <8 x i8> [ [[TMP81]], [[PRED_LOAD_CONTINUE27]] ], [ [[TMP86]], [[PRED_LOAD_IF28]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP88:%.*]] = extractelement <8 x i1> [[TMP7]], i32 5 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP88]], label [[PRED_LOAD_IF30:%.*]], label [[PRED_LOAD_CONTINUE31:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP85:%.*]] = phi <8 x i8> [ [[TMP79]], [[PRED_LOAD_CONTINUE27]] ], [ [[TMP84]], [[PRED_LOAD_IF28]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP86:%.*]] = extractelement <8 x i1> [[TMP5]], i32 5 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP86]], label [[PRED_LOAD_IF30:%.*]], label [[PRED_LOAD_CONTINUE31:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if30: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP89:%.*]] = extractelement <8 x i32> [[TMP57]], i32 5 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP90:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP89]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP91:%.*]] = load i8, i8* [[TMP90]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP92:%.*]] = insertelement <8 x i8> [[TMP87]], i8 [[TMP91]], i32 5 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP87:%.*]] = extractelement <8 x i32> [[TMP55]], i32 5 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP88:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP87]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP89:%.*]] = load i8, i8* [[TMP88]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP90:%.*]] = insertelement <8 x i8> [[TMP85]], i8 [[TMP89]], i32 5 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE31]] ; ENABLED_MASKED_STRIDED: pred.load.continue31: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP93:%.*]] = phi <8 x i8> [ [[TMP87]], [[PRED_LOAD_CONTINUE29]] ], [ [[TMP92]], [[PRED_LOAD_IF30]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP94:%.*]] = extractelement <8 x i1> [[TMP7]], i32 6 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP94]], label [[PRED_LOAD_IF32:%.*]], label [[PRED_LOAD_CONTINUE33:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP91:%.*]] = phi <8 x i8> [ [[TMP85]], [[PRED_LOAD_CONTINUE29]] ], [ [[TMP90]], [[PRED_LOAD_IF30]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP92:%.*]] = extractelement <8 x i1> [[TMP5]], i32 6 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP92]], label [[PRED_LOAD_IF32:%.*]], label [[PRED_LOAD_CONTINUE33:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if32: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP95:%.*]] = extractelement <8 x i32> [[TMP57]], i32 6 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP96:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP95]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP97:%.*]] = load i8, i8* [[TMP96]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP98:%.*]] = insertelement <8 x i8> [[TMP93]], i8 [[TMP97]], i32 6 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP93:%.*]] = extractelement <8 x i32> [[TMP55]], i32 6 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP94:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP93]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP95:%.*]] = load i8, i8* [[TMP94]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP96:%.*]] = insertelement <8 x i8> [[TMP91]], i8 [[TMP95]], i32 6 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE33]] ; ENABLED_MASKED_STRIDED: pred.load.continue33: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP99:%.*]] = phi <8 x i8> [ [[TMP93]], [[PRED_LOAD_CONTINUE31]] ], [ [[TMP98]], [[PRED_LOAD_IF32]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP100:%.*]] = extractelement <8 x i1> [[TMP7]], i32 7 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP100]], label [[PRED_LOAD_IF34:%.*]], label [[PRED_LOAD_CONTINUE35:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP97:%.*]] = phi <8 x i8> [ [[TMP91]], [[PRED_LOAD_CONTINUE31]] ], [ [[TMP96]], [[PRED_LOAD_IF32]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP98:%.*]] = extractelement <8 x i1> [[TMP5]], i32 7 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP98]], label [[PRED_LOAD_IF34:%.*]], label [[PRED_LOAD_CONTINUE35:%.*]] ; ENABLED_MASKED_STRIDED: pred.load.if34: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP101:%.*]] = extractelement <8 x i32> [[TMP57]], i32 7 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP102:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP101]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP103:%.*]] = load i8, i8* [[TMP102]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP104:%.*]] = insertelement <8 x i8> [[TMP99]], i8 [[TMP103]], i32 7 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP99:%.*]] = extractelement <8 x i32> [[TMP55]], i32 7 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP100:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[TMP99]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP101:%.*]] = load i8, i8* [[TMP100]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP102:%.*]] = insertelement <8 x i8> [[TMP97]], i8 [[TMP101]], i32 7 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_LOAD_CONTINUE35]] ; ENABLED_MASKED_STRIDED: pred.load.continue35: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP105:%.*]] = phi <8 x i8> [ [[TMP99]], [[PRED_LOAD_CONTINUE33]] ], [ [[TMP104]], [[PRED_LOAD_IF34]] ] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = icmp slt <8 x i8> [[TMP56]], [[TMP105]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = select <8 x i1> [[TMP106]], <8 x i8> [[TMP105]], <8 x i8> [[TMP56]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = extractelement <8 x i1> [[TMP7]], i32 0 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP108]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP103:%.*]] = phi <8 x i8> [ [[TMP97]], [[PRED_LOAD_CONTINUE33]] ], [ [[TMP102]], [[PRED_LOAD_IF34]] ] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP104:%.*]] = icmp slt <8 x i8> [[TMP54]], [[TMP103]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP105:%.*]] = select <8 x i1> [[TMP104]], <8 x i8> [[TMP103]], <8 x i8> [[TMP54]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = extractelement <8 x i1> [[TMP5]], i32 0 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP106]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] ; ENABLED_MASKED_STRIDED: pred.store.if: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = extractelement <8 x i32> [[TMP8]], i32 0 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP109]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = extractelement <8 x i8> [[TMP107]], i32 0 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP111]], i8* [[TMP110]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = extractelement <8 x i32> [[TMP6]], i32 0 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP107]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = extractelement <8 x i8> [[TMP105]], i32 0 +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP109]], i8* [[TMP108]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = extractelement <8 x i8> [[TMP105]], i32 0 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = sub i8 0, [[TMP110]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = extractelement <8 x i32> [[TMP55]], i32 0 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP112]] +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP111]], i8* [[TMP113]], align 1 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE]] ; ENABLED_MASKED_STRIDED: pred.store.continue: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = extractelement <8 x i1> [[TMP7]], i32 1 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP112]], label [[PRED_STORE_IF36:%.*]], label [[PRED_STORE_CONTINUE37:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = extractelement <8 x i1> [[TMP5]], i32 1 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP114]], label [[PRED_STORE_IF36:%.*]], label [[PRED_STORE_CONTINUE37:%.*]] ; ENABLED_MASKED_STRIDED: pred.store.if36: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = extractelement <8 x i32> [[TMP8]], i32 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP113]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = extractelement <8 x i8> [[TMP107]], i32 1 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP115]], i8* [[TMP114]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = extractelement <8 x i32> [[TMP6]], i32 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP115]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = extractelement <8 x i8> [[TMP105]], i32 1 +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP117]], i8* [[TMP116]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = extractelement <8 x i8> [[TMP105]], i32 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = sub i8 0, [[TMP118]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = extractelement <8 x i32> [[TMP55]], i32 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP120]] +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP119]], i8* [[TMP121]], align 1 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE37]] ; ENABLED_MASKED_STRIDED: pred.store.continue37: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = extractelement <8 x i1> [[TMP7]], i32 2 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP116]], label [[PRED_STORE_IF38:%.*]], label [[PRED_STORE_CONTINUE39:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = extractelement <8 x i1> [[TMP5]], i32 2 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP122]], label [[PRED_STORE_IF38:%.*]], label [[PRED_STORE_CONTINUE39:%.*]] ; ENABLED_MASKED_STRIDED: pred.store.if38: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = extractelement <8 x i32> [[TMP8]], i32 2 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP117]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = extractelement <8 x i8> [[TMP107]], i32 2 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP119]], i8* [[TMP118]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = extractelement <8 x i32> [[TMP6]], i32 2 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP123]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = extractelement <8 x i8> [[TMP105]], i32 2 +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP125]], i8* [[TMP124]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = extractelement <8 x i8> [[TMP105]], i32 2 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = sub i8 0, [[TMP126]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = extractelement <8 x i32> [[TMP55]], i32 2 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP128]] +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP127]], i8* [[TMP129]], align 1 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE39]] ; ENABLED_MASKED_STRIDED: pred.store.continue39: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = extractelement <8 x i1> [[TMP7]], i32 3 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP120]], label [[PRED_STORE_IF40:%.*]], label [[PRED_STORE_CONTINUE41:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = extractelement <8 x i1> [[TMP5]], i32 3 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP130]], label [[PRED_STORE_IF40:%.*]], label [[PRED_STORE_CONTINUE41:%.*]] ; ENABLED_MASKED_STRIDED: pred.store.if40: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = extractelement <8 x i32> [[TMP8]], i32 3 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP121]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = extractelement <8 x i8> [[TMP107]], i32 3 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP123]], i8* [[TMP122]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = extractelement <8 x i32> [[TMP6]], i32 3 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP131]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = extractelement <8 x i8> [[TMP105]], i32 3 +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP133]], i8* [[TMP132]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = extractelement <8 x i8> [[TMP105]], i32 3 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = sub i8 0, [[TMP134]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i32> [[TMP55]], i32 3 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP136]] +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP135]], i8* [[TMP137]], align 1 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE41]] ; ENABLED_MASKED_STRIDED: pred.store.continue41: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = extractelement <8 x i1> [[TMP7]], i32 4 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP124]], label [[PRED_STORE_IF42:%.*]], label [[PRED_STORE_CONTINUE43:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i1> [[TMP5]], i32 4 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP138]], label [[PRED_STORE_IF42:%.*]], label [[PRED_STORE_CONTINUE43:%.*]] ; ENABLED_MASKED_STRIDED: pred.store.if42: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = extractelement <8 x i32> [[TMP8]], i32 4 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP125]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = extractelement <8 x i8> [[TMP107]], i32 4 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP127]], i8* [[TMP126]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = extractelement <8 x i32> [[TMP6]], i32 4 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP139]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i8> [[TMP105]], i32 4 +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP141]], i8* [[TMP140]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i8> [[TMP105]], i32 4 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = sub i8 0, [[TMP142]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i32> [[TMP55]], i32 4 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP144]] +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP143]], i8* [[TMP145]], align 1 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE43]] ; ENABLED_MASKED_STRIDED: pred.store.continue43: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = extractelement <8 x i1> [[TMP7]], i32 5 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP128]], label [[PRED_STORE_IF44:%.*]], label [[PRED_STORE_CONTINUE45:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = extractelement <8 x i1> [[TMP5]], i32 5 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP146]], label [[PRED_STORE_IF44:%.*]], label [[PRED_STORE_CONTINUE45:%.*]] ; ENABLED_MASKED_STRIDED: pred.store.if44: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = extractelement <8 x i32> [[TMP8]], i32 5 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP129]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = extractelement <8 x i8> [[TMP107]], i32 5 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP131]], i8* [[TMP130]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = extractelement <8 x i32> [[TMP6]], i32 5 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP147]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = extractelement <8 x i8> [[TMP105]], i32 5 +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP149]], i8* [[TMP148]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i8> [[TMP105]], i32 5 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = sub i8 0, [[TMP150]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = extractelement <8 x i32> [[TMP55]], i32 5 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP152]] +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP151]], i8* [[TMP153]], align 1 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE45]] ; ENABLED_MASKED_STRIDED: pred.store.continue45: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = extractelement <8 x i1> [[TMP7]], i32 6 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP132]], label [[PRED_STORE_IF46:%.*]], label [[PRED_STORE_CONTINUE47:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i1> [[TMP5]], i32 6 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP154]], label [[PRED_STORE_IF46:%.*]], label [[PRED_STORE_CONTINUE47:%.*]] ; ENABLED_MASKED_STRIDED: pred.store.if46: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = extractelement <8 x i32> [[TMP8]], i32 6 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP133]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = extractelement <8 x i8> [[TMP107]], i32 6 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP135]], i8* [[TMP134]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = extractelement <8 x i32> [[TMP6]], i32 6 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP155]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = extractelement <8 x i8> [[TMP105]], i32 6 +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP157]], i8* [[TMP156]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = extractelement <8 x i8> [[TMP105]], i32 6 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = sub i8 0, [[TMP158]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = extractelement <8 x i32> [[TMP55]], i32 6 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP161:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP160]] +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP159]], i8* [[TMP161]], align 1 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE47]] ; ENABLED_MASKED_STRIDED: pred.store.continue47: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i1> [[TMP7]], i32 7 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP136]], label [[PRED_STORE_IF48:%.*]], label [[PRED_STORE_CONTINUE49:%.*]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = extractelement <8 x i1> [[TMP5]], i32 7 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP162]], label [[PRED_STORE_IF48:%.*]], label [[PRED_STORE_CONTINUE49]] ; ENABLED_MASKED_STRIDED: pred.store.if48: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = extractelement <8 x i32> [[TMP8]], i32 7 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP137]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = extractelement <8 x i8> [[TMP107]], i32 7 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP139]], i8* [[TMP138]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP163:%.*]] = extractelement <8 x i32> [[TMP6]], i32 7 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP164:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP163]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP165:%.*]] = extractelement <8 x i8> [[TMP105]], i32 7 +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP165]], i8* [[TMP164]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP166:%.*]] = extractelement <8 x i8> [[TMP105]], i32 7 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP167:%.*]] = sub i8 0, [[TMP166]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP168:%.*]] = extractelement <8 x i32> [[TMP55]], i32 7 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP169:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP168]] +; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP167]], i8* [[TMP169]], align 1 ; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE49]] ; ENABLED_MASKED_STRIDED: pred.store.continue49: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = sub <8 x i8> zeroinitializer, [[TMP107]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i1> [[TMP7]], i32 0 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP141]], label [[PRED_STORE_IF50:%.*]], label [[PRED_STORE_CONTINUE51:%.*]] -; ENABLED_MASKED_STRIDED: pred.store.if50: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i32> [[TMP57]], i32 0 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP142]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i8> [[TMP140]], i32 0 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP144]], i8* [[TMP143]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE51]] -; ENABLED_MASKED_STRIDED: pred.store.continue51: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = extractelement <8 x i1> [[TMP7]], i32 1 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP145]], label [[PRED_STORE_IF52:%.*]], label [[PRED_STORE_CONTINUE53:%.*]] -; ENABLED_MASKED_STRIDED: pred.store.if52: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = extractelement <8 x i32> [[TMP57]], i32 1 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP146]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = extractelement <8 x i8> [[TMP140]], i32 1 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP148]], i8* [[TMP147]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE53]] -; ENABLED_MASKED_STRIDED: pred.store.continue53: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = extractelement <8 x i1> [[TMP7]], i32 2 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP149]], label [[PRED_STORE_IF54:%.*]], label [[PRED_STORE_CONTINUE55:%.*]] -; ENABLED_MASKED_STRIDED: pred.store.if54: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i32> [[TMP57]], i32 2 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP150]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = extractelement <8 x i8> [[TMP140]], i32 2 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP152]], i8* [[TMP151]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE55]] -; ENABLED_MASKED_STRIDED: pred.store.continue55: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = extractelement <8 x i1> [[TMP7]], i32 3 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP153]], label [[PRED_STORE_IF56:%.*]], label [[PRED_STORE_CONTINUE57:%.*]] -; ENABLED_MASKED_STRIDED: pred.store.if56: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i32> [[TMP57]], i32 3 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP154]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = extractelement <8 x i8> [[TMP140]], i32 3 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP156]], i8* [[TMP155]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE57]] -; ENABLED_MASKED_STRIDED: pred.store.continue57: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = extractelement <8 x i1> [[TMP7]], i32 4 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP157]], label [[PRED_STORE_IF58:%.*]], label [[PRED_STORE_CONTINUE59:%.*]] -; ENABLED_MASKED_STRIDED: pred.store.if58: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = extractelement <8 x i32> [[TMP57]], i32 4 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP158]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = extractelement <8 x i8> [[TMP140]], i32 4 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP160]], i8* [[TMP159]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE59]] -; ENABLED_MASKED_STRIDED: pred.store.continue59: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP161:%.*]] = extractelement <8 x i1> [[TMP7]], i32 5 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP161]], label [[PRED_STORE_IF60:%.*]], label [[PRED_STORE_CONTINUE61:%.*]] -; ENABLED_MASKED_STRIDED: pred.store.if60: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = extractelement <8 x i32> [[TMP57]], i32 5 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP163:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP162]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP164:%.*]] = extractelement <8 x i8> [[TMP140]], i32 5 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP164]], i8* [[TMP163]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE61]] -; ENABLED_MASKED_STRIDED: pred.store.continue61: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP165:%.*]] = extractelement <8 x i1> [[TMP7]], i32 6 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP165]], label [[PRED_STORE_IF62:%.*]], label [[PRED_STORE_CONTINUE63:%.*]] -; ENABLED_MASKED_STRIDED: pred.store.if62: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP166:%.*]] = extractelement <8 x i32> [[TMP57]], i32 6 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP167:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP166]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP168:%.*]] = extractelement <8 x i8> [[TMP140]], i32 6 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP168]], i8* [[TMP167]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE63]] -; ENABLED_MASKED_STRIDED: pred.store.continue63: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP169:%.*]] = extractelement <8 x i1> [[TMP7]], i32 7 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP169]], label [[PRED_STORE_IF64:%.*]], label [[PRED_STORE_CONTINUE65]] -; ENABLED_MASKED_STRIDED: pred.store.if64: -; ENABLED_MASKED_STRIDED-NEXT: [[TMP170:%.*]] = extractelement <8 x i32> [[TMP57]], i32 7 -; ENABLED_MASKED_STRIDED-NEXT: [[TMP171:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP170]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP172:%.*]] = extractelement <8 x i8> [[TMP140]], i32 7 -; ENABLED_MASKED_STRIDED-NEXT: store i8 [[TMP172]], i8* [[TMP171]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE65]] -; ENABLED_MASKED_STRIDED: pred.store.continue65: ; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 ; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], -; ENABLED_MASKED_STRIDED-NEXT: [[TMP173:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP173]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP170:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024 +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP170]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; ENABLED_MASKED_STRIDED: for.body: ; ENABLED_MASKED_STRIDED-NEXT: [[IX_024:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_INC:%.*]] ], [ 1024, [[ENTRY:%.*]] ] ; ENABLED_MASKED_STRIDED-NEXT: [[CMP1:%.*]] = icmp ugt i32 [[IX_024]], [[CONV]] @@ -2186,12 +2105,12 @@ ; ENABLED_MASKED_STRIDED: if.then: ; ENABLED_MASKED_STRIDED-NEXT: [[MUL:%.*]] = shl nuw nsw i32 [[IX_024]], 1 ; ENABLED_MASKED_STRIDED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[MUL]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP174:%.*]] = load i8, i8* [[ARRAYIDX]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[TMP171:%.*]] = load i8, i8* [[ARRAYIDX]], align 1 ; ENABLED_MASKED_STRIDED-NEXT: [[ADD:%.*]] = or i32 [[MUL]], 1 ; ENABLED_MASKED_STRIDED-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 [[ADD]] -; ENABLED_MASKED_STRIDED-NEXT: [[TMP175:%.*]] = load i8, i8* [[ARRAYIDX4]], align 1 -; ENABLED_MASKED_STRIDED-NEXT: [[CMP_I:%.*]] = icmp slt i8 [[TMP174]], [[TMP175]] -; ENABLED_MASKED_STRIDED-NEXT: [[SPEC_SELECT_I:%.*]] = select i1 [[CMP_I]], i8 [[TMP175]], i8 [[TMP174]] +; ENABLED_MASKED_STRIDED-NEXT: [[TMP172:%.*]] = load i8, i8* [[ARRAYIDX4]], align 1 +; ENABLED_MASKED_STRIDED-NEXT: [[CMP_I:%.*]] = icmp slt i8 [[TMP171]], [[TMP172]] +; ENABLED_MASKED_STRIDED-NEXT: [[SPEC_SELECT_I:%.*]] = select i1 [[CMP_I]], i8 [[TMP172]], i8 [[TMP171]] ; ENABLED_MASKED_STRIDED-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[MUL]] ; ENABLED_MASKED_STRIDED-NEXT: store i8 [[SPEC_SELECT_I]], i8* [[ARRAYIDX6]], align 1 ; ENABLED_MASKED_STRIDED-NEXT: [[SUB:%.*]] = sub i8 0, [[SPEC_SELECT_I]] @@ -2201,7 +2120,7 @@ ; ENABLED_MASKED_STRIDED: for.inc: ; ENABLED_MASKED_STRIDED-NEXT: [[INC]] = add nsw i32 [[IX_024]], -1 ; ENABLED_MASKED_STRIDED-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 0 -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]] +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] ; ENABLED_MASKED_STRIDED: for.end: ; ENABLED_MASKED_STRIDED-NEXT: ret void ; @@ -2280,8 +2199,8 @@ ; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer ; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; DISABLED_MASKED_STRIDED: vector.body: -; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE62:%.*]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ , [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE62]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE46:%.*]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ , [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE46]] ] ; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp sgt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]] ; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] ; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], @@ -2456,148 +2375,115 @@ ; DISABLED_MASKED_STRIDED-NEXT: [[TMP105:%.*]] = getelementptr inbounds i8, i8* [[Q:%.*]], i32 [[TMP104]] ; DISABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = extractelement <8 x i8> [[TMP102]], i32 0 ; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP106]], i8* [[TMP105]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = extractelement <8 x i8> [[TMP102]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = sub i8 0, [[TMP107]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = extractelement <8 x i32> [[TMP52]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP109]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP108]], i8* [[TMP110]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE]] ; DISABLED_MASKED_STRIDED: pred.store.continue: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = extractelement <8 x i1> [[TMP3]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP107]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = extractelement <8 x i1> [[TMP3]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP111]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if33: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = extractelement <8 x i32> [[TMP2]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP108]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = extractelement <8 x i8> [[TMP102]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP110]], i8* [[TMP109]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = extractelement <8 x i32> [[TMP2]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP112]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = extractelement <8 x i8> [[TMP102]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP114]], i8* [[TMP113]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = extractelement <8 x i8> [[TMP102]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = sub i8 0, [[TMP115]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = extractelement <8 x i32> [[TMP52]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP117]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP116]], i8* [[TMP118]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE34]] ; DISABLED_MASKED_STRIDED: pred.store.continue34: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = extractelement <8 x i1> [[TMP3]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP111]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = extractelement <8 x i1> [[TMP3]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP119]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if35: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = extractelement <8 x i32> [[TMP2]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP112]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = extractelement <8 x i8> [[TMP102]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP114]], i8* [[TMP113]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = extractelement <8 x i32> [[TMP2]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP120]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = extractelement <8 x i8> [[TMP102]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP122]], i8* [[TMP121]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = extractelement <8 x i8> [[TMP102]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = sub i8 0, [[TMP123]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = extractelement <8 x i32> [[TMP52]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP125]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP124]], i8* [[TMP126]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE36]] ; DISABLED_MASKED_STRIDED: pred.store.continue36: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = extractelement <8 x i1> [[TMP3]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP115]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = extractelement <8 x i1> [[TMP3]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP127]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if37: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = extractelement <8 x i32> [[TMP2]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP116]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = extractelement <8 x i8> [[TMP102]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP118]], i8* [[TMP117]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = extractelement <8 x i32> [[TMP2]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP128]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = extractelement <8 x i8> [[TMP102]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP130]], i8* [[TMP129]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = extractelement <8 x i8> [[TMP102]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = sub i8 0, [[TMP131]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = extractelement <8 x i32> [[TMP52]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP133]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP132]], i8* [[TMP134]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE38]] ; DISABLED_MASKED_STRIDED: pred.store.continue38: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = extractelement <8 x i1> [[TMP3]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP119]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = extractelement <8 x i1> [[TMP3]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP135]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if39: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = extractelement <8 x i32> [[TMP2]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP120]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = extractelement <8 x i8> [[TMP102]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP122]], i8* [[TMP121]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i32> [[TMP2]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP136]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i8> [[TMP102]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP138]], i8* [[TMP137]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = extractelement <8 x i8> [[TMP102]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = sub i8 0, [[TMP139]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i32> [[TMP52]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP141]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP140]], i8* [[TMP142]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE40]] ; DISABLED_MASKED_STRIDED: pred.store.continue40: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = extractelement <8 x i1> [[TMP3]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP123]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = extractelement <8 x i1> [[TMP3]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP143]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if41: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = extractelement <8 x i32> [[TMP2]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP124]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = extractelement <8 x i8> [[TMP102]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP126]], i8* [[TMP125]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i32> [[TMP2]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP144]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = extractelement <8 x i8> [[TMP102]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP146]], i8* [[TMP145]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = extractelement <8 x i8> [[TMP102]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = sub i8 0, [[TMP147]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = extractelement <8 x i32> [[TMP52]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP149]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP148]], i8* [[TMP150]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE42]] ; DISABLED_MASKED_STRIDED: pred.store.continue42: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = extractelement <8 x i1> [[TMP3]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP127]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = extractelement <8 x i1> [[TMP3]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP151]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if43: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = extractelement <8 x i32> [[TMP2]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP128]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = extractelement <8 x i8> [[TMP102]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP130]], i8* [[TMP129]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = extractelement <8 x i32> [[TMP2]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP152]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i8> [[TMP102]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP154]], i8* [[TMP153]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = extractelement <8 x i8> [[TMP102]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = sub i8 0, [[TMP155]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = extractelement <8 x i32> [[TMP52]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP157]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP156]], i8* [[TMP158]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE44]] ; DISABLED_MASKED_STRIDED: pred.store.continue44: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = extractelement <8 x i1> [[TMP3]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP131]], label [[PRED_STORE_IF45:%.*]], label [[PRED_STORE_CONTINUE46:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = extractelement <8 x i1> [[TMP3]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP159]], label [[PRED_STORE_IF45:%.*]], label [[PRED_STORE_CONTINUE46]] ; DISABLED_MASKED_STRIDED: pred.store.if45: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = extractelement <8 x i32> [[TMP2]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP132]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = extractelement <8 x i8> [[TMP102]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP134]], i8* [[TMP133]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE46]] -; DISABLED_MASKED_STRIDED: pred.store.continue46: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = sub <8 x i8> zeroinitializer, [[TMP102]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i1> [[TMP3]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP136]], label [[PRED_STORE_IF47:%.*]], label [[PRED_STORE_CONTINUE48:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if47: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = extractelement <8 x i32> [[TMP52]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP137]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = extractelement <8 x i8> [[TMP135]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP139]], i8* [[TMP138]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE48]] -; DISABLED_MASKED_STRIDED: pred.store.continue48: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = extractelement <8 x i1> [[TMP3]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP140]], label [[PRED_STORE_IF49:%.*]], label [[PRED_STORE_CONTINUE50:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if49: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i32> [[TMP52]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP141]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = extractelement <8 x i8> [[TMP135]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP143]], i8* [[TMP142]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE50]] -; DISABLED_MASKED_STRIDED: pred.store.continue50: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i1> [[TMP3]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP144]], label [[PRED_STORE_IF51:%.*]], label [[PRED_STORE_CONTINUE52:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if51: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = extractelement <8 x i32> [[TMP52]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP145]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = extractelement <8 x i8> [[TMP135]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP147]], i8* [[TMP146]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE52]] -; DISABLED_MASKED_STRIDED: pred.store.continue52: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = extractelement <8 x i1> [[TMP3]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP148]], label [[PRED_STORE_IF53:%.*]], label [[PRED_STORE_CONTINUE54:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if53: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = extractelement <8 x i32> [[TMP52]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP149]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = extractelement <8 x i8> [[TMP135]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP151]], i8* [[TMP150]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE54]] -; DISABLED_MASKED_STRIDED: pred.store.continue54: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = extractelement <8 x i1> [[TMP3]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP152]], label [[PRED_STORE_IF55:%.*]], label [[PRED_STORE_CONTINUE56:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if55: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = extractelement <8 x i32> [[TMP52]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP153]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = extractelement <8 x i8> [[TMP135]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP155]], i8* [[TMP154]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE56]] -; DISABLED_MASKED_STRIDED: pred.store.continue56: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = extractelement <8 x i1> [[TMP3]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP156]], label [[PRED_STORE_IF57:%.*]], label [[PRED_STORE_CONTINUE58:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if57: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = extractelement <8 x i32> [[TMP52]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP157]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = extractelement <8 x i8> [[TMP135]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP159]], i8* [[TMP158]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE58]] -; DISABLED_MASKED_STRIDED: pred.store.continue58: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = extractelement <8 x i1> [[TMP3]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP160]], label [[PRED_STORE_IF59:%.*]], label [[PRED_STORE_CONTINUE60:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if59: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP161:%.*]] = extractelement <8 x i32> [[TMP52]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP161]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP163:%.*]] = extractelement <8 x i8> [[TMP135]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP163]], i8* [[TMP162]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE60]] -; DISABLED_MASKED_STRIDED: pred.store.continue60: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP164:%.*]] = extractelement <8 x i1> [[TMP3]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP164]], label [[PRED_STORE_IF61:%.*]], label [[PRED_STORE_CONTINUE62]] -; DISABLED_MASKED_STRIDED: pred.store.if61: +; DISABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = extractelement <8 x i32> [[TMP2]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP161:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP160]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = extractelement <8 x i8> [[TMP102]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP162]], i8* [[TMP161]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP163:%.*]] = extractelement <8 x i8> [[TMP102]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP164:%.*]] = sub i8 0, [[TMP163]] ; DISABLED_MASKED_STRIDED-NEXT: [[TMP165:%.*]] = extractelement <8 x i32> [[TMP52]], i32 7 ; DISABLED_MASKED_STRIDED-NEXT: [[TMP166:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP165]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP167:%.*]] = extractelement <8 x i8> [[TMP135]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP167]], i8* [[TMP166]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE62]] -; DISABLED_MASKED_STRIDED: pred.store.continue62: +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP164]], i8* [[TMP166]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE46]] +; DISABLED_MASKED_STRIDED: pred.store.continue46: ; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8 ; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], -; DISABLED_MASKED_STRIDED-NEXT: [[TMP168:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP168]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP167:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP167]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; DISABLED_MASKED_STRIDED: for.end: ; DISABLED_MASKED_STRIDED-NEXT: ret void ; @@ -2639,7 +2525,7 @@ ; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8 ; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], ; ENABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP13]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP13]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] ; ENABLED_MASKED_STRIDED: for.end: ; ENABLED_MASKED_STRIDED-NEXT: ret void ; @@ -2722,8 +2608,8 @@ ; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer ; DISABLED_MASKED_STRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; DISABLED_MASKED_STRIDED: vector.body: -; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE60:%.*]] ] -; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ , [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE60]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE44:%.*]] ] +; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ , [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE44]] ] ; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]] ; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], ; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP0]], i32 0 @@ -2896,148 +2782,115 @@ ; DISABLED_MASKED_STRIDED-NEXT: [[TMP103:%.*]] = getelementptr inbounds i8, i8* [[Q:%.*]], i32 [[TMP102]] ; DISABLED_MASKED_STRIDED-NEXT: [[TMP104:%.*]] = extractelement <8 x i8> [[TMP100]], i32 0 ; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP104]], i8* [[TMP103]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP105:%.*]] = extractelement <8 x i8> [[TMP100]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = sub i8 0, [[TMP105]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = extractelement <8 x i32> [[TMP50]], i32 0 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP107]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP106]], i8* [[TMP108]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE]] ; DISABLED_MASKED_STRIDED: pred.store.continue: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP105:%.*]] = extractelement <8 x i1> [[TMP0]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP105]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE32:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = extractelement <8 x i1> [[TMP0]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP109]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE32:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if31: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP106:%.*]] = extractelement <8 x i32> [[TMP1]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP107:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP106]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP108:%.*]] = extractelement <8 x i8> [[TMP100]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP108]], i8* [[TMP107]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = extractelement <8 x i32> [[TMP1]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP110]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = extractelement <8 x i8> [[TMP100]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP112]], i8* [[TMP111]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = extractelement <8 x i8> [[TMP100]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = sub i8 0, [[TMP113]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = extractelement <8 x i32> [[TMP50]], i32 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP115]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP114]], i8* [[TMP116]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE32]] ; DISABLED_MASKED_STRIDED: pred.store.continue32: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP109:%.*]] = extractelement <8 x i1> [[TMP0]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP109]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = extractelement <8 x i1> [[TMP0]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP117]], label [[PRED_STORE_IF33:%.*]], label [[PRED_STORE_CONTINUE34:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if33: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP110:%.*]] = extractelement <8 x i32> [[TMP1]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP111:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP110]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP112:%.*]] = extractelement <8 x i8> [[TMP100]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP112]], i8* [[TMP111]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = extractelement <8 x i32> [[TMP1]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP118]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = extractelement <8 x i8> [[TMP100]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP120]], i8* [[TMP119]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = extractelement <8 x i8> [[TMP100]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = sub i8 0, [[TMP121]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = extractelement <8 x i32> [[TMP50]], i32 2 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP123]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP122]], i8* [[TMP124]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE34]] ; DISABLED_MASKED_STRIDED: pred.store.continue34: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP113:%.*]] = extractelement <8 x i1> [[TMP0]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP113]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = extractelement <8 x i1> [[TMP0]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP125]], label [[PRED_STORE_IF35:%.*]], label [[PRED_STORE_CONTINUE36:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if35: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP114:%.*]] = extractelement <8 x i32> [[TMP1]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP115:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP114]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP116:%.*]] = extractelement <8 x i8> [[TMP100]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP116]], i8* [[TMP115]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = extractelement <8 x i32> [[TMP1]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP126]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = extractelement <8 x i8> [[TMP100]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP128]], i8* [[TMP127]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = extractelement <8 x i8> [[TMP100]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = sub i8 0, [[TMP129]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = extractelement <8 x i32> [[TMP50]], i32 3 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP131]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP130]], i8* [[TMP132]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE36]] ; DISABLED_MASKED_STRIDED: pred.store.continue36: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP117:%.*]] = extractelement <8 x i1> [[TMP0]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP117]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = extractelement <8 x i1> [[TMP0]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP133]], label [[PRED_STORE_IF37:%.*]], label [[PRED_STORE_CONTINUE38:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if37: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP118:%.*]] = extractelement <8 x i32> [[TMP1]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP119:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP118]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP120:%.*]] = extractelement <8 x i8> [[TMP100]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP120]], i8* [[TMP119]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = extractelement <8 x i32> [[TMP1]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP134]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = extractelement <8 x i8> [[TMP100]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP136]], i8* [[TMP135]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = extractelement <8 x i8> [[TMP100]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = sub i8 0, [[TMP137]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = extractelement <8 x i32> [[TMP50]], i32 4 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP139]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP138]], i8* [[TMP140]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE38]] ; DISABLED_MASKED_STRIDED: pred.store.continue38: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP121:%.*]] = extractelement <8 x i1> [[TMP0]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP121]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i1> [[TMP0]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP141]], label [[PRED_STORE_IF39:%.*]], label [[PRED_STORE_CONTINUE40:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if39: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP122:%.*]] = extractelement <8 x i32> [[TMP1]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP123:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP122]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP124:%.*]] = extractelement <8 x i8> [[TMP100]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP124]], i8* [[TMP123]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i32> [[TMP1]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP142]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = extractelement <8 x i8> [[TMP100]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP144]], i8* [[TMP143]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = extractelement <8 x i8> [[TMP100]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = sub i8 0, [[TMP145]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = extractelement <8 x i32> [[TMP50]], i32 5 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP147]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP146]], i8* [[TMP148]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE40]] ; DISABLED_MASKED_STRIDED: pred.store.continue40: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP125:%.*]] = extractelement <8 x i1> [[TMP0]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP125]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = extractelement <8 x i1> [[TMP0]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP149]], label [[PRED_STORE_IF41:%.*]], label [[PRED_STORE_CONTINUE42:%.*]] ; DISABLED_MASKED_STRIDED: pred.store.if41: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP126:%.*]] = extractelement <8 x i32> [[TMP1]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP127:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP126]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP128:%.*]] = extractelement <8 x i8> [[TMP100]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP128]], i8* [[TMP127]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i32> [[TMP1]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP150]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = extractelement <8 x i8> [[TMP100]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP152]], i8* [[TMP151]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = extractelement <8 x i8> [[TMP100]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = sub i8 0, [[TMP153]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = extractelement <8 x i32> [[TMP50]], i32 6 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP155]] +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP154]], i8* [[TMP156]], align 1 ; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE42]] ; DISABLED_MASKED_STRIDED: pred.store.continue42: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP129:%.*]] = extractelement <8 x i1> [[TMP0]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP129]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44:%.*]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = extractelement <8 x i1> [[TMP0]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP157]], label [[PRED_STORE_IF43:%.*]], label [[PRED_STORE_CONTINUE44]] ; DISABLED_MASKED_STRIDED: pred.store.if43: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP130:%.*]] = extractelement <8 x i32> [[TMP1]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP131:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP130]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP132:%.*]] = extractelement <8 x i8> [[TMP100]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP132]], i8* [[TMP131]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE44]] -; DISABLED_MASKED_STRIDED: pred.store.continue44: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP133:%.*]] = sub <8 x i8> zeroinitializer, [[TMP100]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP134:%.*]] = extractelement <8 x i1> [[TMP0]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP134]], label [[PRED_STORE_IF45:%.*]], label [[PRED_STORE_CONTINUE46:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if45: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP135:%.*]] = extractelement <8 x i32> [[TMP50]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP136:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP135]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP137:%.*]] = extractelement <8 x i8> [[TMP133]], i32 0 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP137]], i8* [[TMP136]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE46]] -; DISABLED_MASKED_STRIDED: pred.store.continue46: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP138:%.*]] = extractelement <8 x i1> [[TMP0]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP138]], label [[PRED_STORE_IF47:%.*]], label [[PRED_STORE_CONTINUE48:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if47: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP139:%.*]] = extractelement <8 x i32> [[TMP50]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP140:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP139]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP141:%.*]] = extractelement <8 x i8> [[TMP133]], i32 1 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP141]], i8* [[TMP140]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE48]] -; DISABLED_MASKED_STRIDED: pred.store.continue48: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP142:%.*]] = extractelement <8 x i1> [[TMP0]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP142]], label [[PRED_STORE_IF49:%.*]], label [[PRED_STORE_CONTINUE50:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if49: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP143:%.*]] = extractelement <8 x i32> [[TMP50]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP144:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP143]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP145:%.*]] = extractelement <8 x i8> [[TMP133]], i32 2 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP145]], i8* [[TMP144]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE50]] -; DISABLED_MASKED_STRIDED: pred.store.continue50: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP146:%.*]] = extractelement <8 x i1> [[TMP0]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP146]], label [[PRED_STORE_IF51:%.*]], label [[PRED_STORE_CONTINUE52:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if51: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP147:%.*]] = extractelement <8 x i32> [[TMP50]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP148:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP147]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP149:%.*]] = extractelement <8 x i8> [[TMP133]], i32 3 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP149]], i8* [[TMP148]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE52]] -; DISABLED_MASKED_STRIDED: pred.store.continue52: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP150:%.*]] = extractelement <8 x i1> [[TMP0]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP150]], label [[PRED_STORE_IF53:%.*]], label [[PRED_STORE_CONTINUE54:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if53: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP151:%.*]] = extractelement <8 x i32> [[TMP50]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP152:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP151]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP153:%.*]] = extractelement <8 x i8> [[TMP133]], i32 4 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP153]], i8* [[TMP152]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE54]] -; DISABLED_MASKED_STRIDED: pred.store.continue54: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP154:%.*]] = extractelement <8 x i1> [[TMP0]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP154]], label [[PRED_STORE_IF55:%.*]], label [[PRED_STORE_CONTINUE56:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if55: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP155:%.*]] = extractelement <8 x i32> [[TMP50]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP156:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP155]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = extractelement <8 x i8> [[TMP133]], i32 5 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP157]], i8* [[TMP156]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE56]] -; DISABLED_MASKED_STRIDED: pred.store.continue56: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = extractelement <8 x i1> [[TMP0]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP158]], label [[PRED_STORE_IF57:%.*]], label [[PRED_STORE_CONTINUE58:%.*]] -; DISABLED_MASKED_STRIDED: pred.store.if57: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = extractelement <8 x i32> [[TMP50]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP159]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP161:%.*]] = extractelement <8 x i8> [[TMP133]], i32 6 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP161]], i8* [[TMP160]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE58]] -; DISABLED_MASKED_STRIDED: pred.store.continue58: -; DISABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = extractelement <8 x i1> [[TMP0]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP162]], label [[PRED_STORE_IF59:%.*]], label [[PRED_STORE_CONTINUE60]] -; DISABLED_MASKED_STRIDED: pred.store.if59: +; DISABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = extractelement <8 x i32> [[TMP1]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP158]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = extractelement <8 x i8> [[TMP100]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP160]], i8* [[TMP159]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP161:%.*]] = extractelement <8 x i8> [[TMP100]], i32 7 +; DISABLED_MASKED_STRIDED-NEXT: [[TMP162:%.*]] = sub i8 0, [[TMP161]] ; DISABLED_MASKED_STRIDED-NEXT: [[TMP163:%.*]] = extractelement <8 x i32> [[TMP50]], i32 7 ; DISABLED_MASKED_STRIDED-NEXT: [[TMP164:%.*]] = getelementptr inbounds i8, i8* [[Q]], i32 [[TMP163]] -; DISABLED_MASKED_STRIDED-NEXT: [[TMP165:%.*]] = extractelement <8 x i8> [[TMP133]], i32 7 -; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP165]], i8* [[TMP164]], align 1 -; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE60]] -; DISABLED_MASKED_STRIDED: pred.store.continue60: +; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP162]], i8* [[TMP164]], align 1 +; DISABLED_MASKED_STRIDED-NEXT: br label [[PRED_STORE_CONTINUE44]] +; DISABLED_MASKED_STRIDED: pred.store.continue44: ; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8 ; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], -; DISABLED_MASKED_STRIDED-NEXT: [[TMP166:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP166]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] +; DISABLED_MASKED_STRIDED-NEXT: [[TMP165:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP165]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] ; DISABLED_MASKED_STRIDED: for.end: ; DISABLED_MASKED_STRIDED-NEXT: ret void ; @@ -3076,7 +2929,7 @@ ; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> [[INTERLEAVED_VEC]], <16 x i8>* [[TMP10]], i32 1, <16 x i1> [[INTERLEAVED_MASK]]) ; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8 ; ENABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP11]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] +; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP11]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] ; ENABLED_MASKED_STRIDED: for.end: ; ENABLED_MASKED_STRIDED-NEXT: ret void ; diff --git a/llvm/test/Transforms/LoopVectorize/optsize.ll b/llvm/test/Transforms/LoopVectorize/optsize.ll --- a/llvm/test/Transforms/LoopVectorize/optsize.ll +++ b/llvm/test/Transforms/LoopVectorize/optsize.ll @@ -8,6 +8,8 @@ ; RUN: opt < %s -passes='require,loop-vectorize' -pgso -S | FileCheck %s -check-prefix=PGSO ; RUN: opt < %s -passes='require,loop-vectorize' -pgso=false -S | FileCheck %s -check-prefix=NPGSO +; XFAIL: * + target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128" @tab = common global [32 x i8] zeroinitializer, align 1 diff --git a/llvm/test/Transforms/LoopVectorize/tripcount.ll b/llvm/test/Transforms/LoopVectorize/tripcount.ll --- a/llvm/test/Transforms/LoopVectorize/tripcount.ll +++ b/llvm/test/Transforms/LoopVectorize/tripcount.ll @@ -3,6 +3,8 @@ ; REQUIRES: asserts ; RUN: opt < %s -loop-vectorize -loop-vectorize-with-block-frequency -S | FileCheck %s +; XFAIL: * + target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128" @tab = common global [32 x i8] zeroinitializer, align 1 diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -622,77 +622,79 @@ ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: loop: -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%iv> -; CHECK-NEXT: Successor(s): loop.0 +; CHECK-NEXT: loop: +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next +; CHECK-NEXT: EMIT vp<%2> = icmp ule ir<%iv> vp<%0> +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%iv> +; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: -; CHECK-NEXT: loop.0: -; CHECK-NEXT: Successor(s): loop.1 +; CHECK-NEXT: loop.0: +; CHECK-NEXT: Successor(s): loop.1 ; CHECK-EMPTY: -; CHECK-NEXT: loop.1: -; CHECK-NEXT: Successor(s): loop.2 +; CHECK-NEXT: loop.1: +; CHECK-NEXT: Successor(s): loop.2 ; CHECK-EMPTY: -; CHECK-NEXT: loop.2: -; CHECK-NEXT: Successor(s): pred.store +; CHECK-NEXT: loop.2: +; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (loop) +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%2> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-NEXT: CondBit: vp<%2> (loop) ; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, ir<%iv> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> -; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr ir<@c>, ir<0>, ir<%iv> -; CHECK-NEXT: REPLICATE store ir<%lv.a>, ir<%gep.c> -; CHECK-NEXT: REPLICATE store ir<%lv.b>, ir<%gep.a> -; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, ir<%iv> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> +; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr ir<@c>, ir<0>, ir<%iv> +; CHECK-NEXT: REPLICATE store ir<%lv.a>, ir<%gep.c> +; CHECK-NEXT: REPLICATE store ir<%lv.b>, ir<%gep.a> +; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.b> +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%10> = ir<%lv.a> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%11> = ir<%lv.b> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.3 +; CHECK-EMPTY: +; CHECK-NEXT: loop.3: +; CHECK-NEXT: WIDEN ir<%c.0> = icmp ir<%iv>, ir<%j> +; CHECK-NEXT: Successor(s): then.0 +; CHECK-EMPTY: +; CHECK-NEXT: then.0: +; CHECK-NEXT: EMIT vp<%13> = select vp<%2> ir<%c.0> ir +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%13> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-NEXT: CondBit: vp<%13> (then.0) +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%mul> = mul vp<%10>, vp<%11> +; CHECK-NEXT: REPLICATE ir<%gep.c.1> = getelementptr ir<@c>, ir<0>, ir<%iv> +; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.c.1> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): then.0.0 +; CHECK-EMPTY: +; CHECK-NEXT: then.0.0: +; CHECK-NEXT: Successor(s): latch +; CHECK-EMPTY: +; CHECK-NEXT: latch: +; CHECK-NEXT: CLONE ir<%large> = icmp ir<%iv>, ir<8> +; CHECK-NEXT: CLONE ir<%exitcond> = icmp ir<%iv>, ir<%k> ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.3 -; CHECK-EMPTY: -; CHECK-NEXT: loop.3: -; CHECK-NEXT: WIDEN ir<%c.0> = icmp ir<%iv>, ir<%j> -; CHECK-NEXT: Successor(s): then.0 -; CHECK-EMPTY: -; CHECK-NEXT: then.0: -; CHECK-NEXT: WIDEN ir<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]> -; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]> ir<%c.0> ir -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK2]]> (then.0) -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.c.1> = getelementptr ir<@c>, ir<0>, ir<%iv> -; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.c.1> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): then.0.0 -; CHECK-EMPTY: -; CHECK-NEXT: then.0.0: -; CHECK-NEXT: Successor(s): latch -; CHECK-EMPTY: -; CHECK-NEXT: latch: -; CHECK-NEXT: CLONE ir<%large> = icmp ir<%iv>, ir<8> -; CHECK-NEXT: CLONE ir<%exitcond> = icmp ir<%iv>, ir<%k> ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; + entry: br label %loop