This is an archive of the discontinued LLVM Phabricator instance.

Differential D85410

[ARM][MVE] Allow loops containing strides != 1 to be tail predicated with gather/scatters enabled
ClosedPublic

Authored by anwel on Aug 6 2020, 1:27 AM.

Details

Reviewers
dmgreen
SjoerdMeijer
samparker
Commits
rG8048068c3ec5: [ARM][MVE] Allow tail predication for strides !=1 with gather/scatters
Summary

There is a stride check in ARMTargetTransformInfo that decides whether a loop cannot be tail predicated by checking whether the strides in it are different from 1. To enable tail predication for loops containing gather/scatters, this patch takes a more detailed approach if the EnableMaskedGatherScatters flag is true, and also adds some more detailed debug messages.

Diff Detail

Event Timeline

anwel created this revision.Aug 6 2020, 1:27 AM
Herald added subscribers: llvm-commits, danielkiss, hiraditya, kristof.beyls. · View Herald TranscriptAug 6 2020, 1:27 AM
anwel requested review of this revision.Aug 6 2020, 1:27 AM
samparker added inline comments.Aug 6 2020, 1:58 AM
llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
1634

else ifs?

1639

I think this would be easier to read if this was organised in stride order and separating the gather/scatter from the consecutive accesses. So when !EnableMaskedGatherScatter, getPtrStride should only ever be 1, right? So I don't think we have to track the 'NextStride' business.

dmgreen added inline comments.Aug 6 2020, 2:09 AM
llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
1634

A load or a store can be vld2 or a vst2, neither of which can be tail folded unfortunately.

anwel updated this revision to Diff 285960.Aug 17 2020, 4:21 AM
anwel marked 3 inline comments as done.

Widened the range of allowed strides to also include loop invariant expressions.

llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
1634

Good point.

1634

Right, I forgot about the vstr2's. In that case we can never allow a stride of 2 here, as the only instructions that get us here are loads and stores.

1639

Good point. We may as well get rid of that.

dmgreen added inline comments.Aug 17 2020, 6:55 AM
llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
1634–1639

Can you change this condition to something like

if (NextStride == -1 || (NextStride == 2 && MVEMaxSupportedInterleaveFactor >= 2) || (NextStride == 4 && MVEMaxSupportedInterleaveFactor >= 4))

That should hopefully make it more futureproof, and specifically rule out reverse loads even if the vectorizer changes to support them.

1649–1650

Perhaps

if (auto AR = dyn_cast<SCEVAddRecExpr>(PtrScev)) {
anwel updated this revision to Diff 286754.Aug 20 2020, 2:45 AM
anwel marked 2 inline comments as done.

I made the NextStride == 2 condition more future proof as requested by Dave's comment. This required to make the MVEMaxSupportedInterleaveFactor from ARMISelLowering non-static.

llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
1634–1639

Of course I can, that sounds like a good idea.

1649–1650

Sure

dmgreen accepted this revision.Aug 20 2020, 6:57 AM

Thanks. Looks good to me.

llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
31

Do you need to include this?

This revision is now accepted and ready to land.Aug 20 2020, 6:57 AM
anwel marked an inline comment as done.Aug 24 2020, 3:59 AM
anwel added inline comments.
llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
31

Oops, no. Will remove for commit.

Closed by commit rG8048068c3ec5: [ARM][MVE] Allow tail predication for strides !=1 with gather/scatters (authored by anwel). · Explain WhyAug 24 2020, 5:55 AM
This revision was automatically updated to reflect the committed changes.
anwel marked an inline comment as done.
anwel added a commit: rG8048068c3ec5: [ARM][MVE] Allow tail predication for strides !=1 with gather/scatters.

Revision Contents

PathSize
llvm/
lib/
Target/
ARM/
ARMISelLowering.cpp
ARMISelLowering.cpp
2 lines
ARMTargetTransformInfo.cpp
ARMTargetTransformInfo.cpp
40 lines
llvm/
test/
Transforms/
LoopVectorize/
ARM/
dev/
391 lines

Diff 286754

llvm/lib/Target/ARM/ARMISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 137 Lines • ▼ Show 20 Linesstatic cl::opt<unsigned> ConstpoolPromotionMaxSize(
"arm-promote-constant-max-size", cl::Hidden, "arm-promote-constant-max-size", cl::Hidden,
cl::desc("Maximum size of constant to promote into a constant pool"), cl::desc("Maximum size of constant to promote into a constant pool"),
cl::init(64)); cl::init(64));
static cl::opt<unsigned> ConstpoolPromotionMaxTotal( static cl::opt<unsigned> ConstpoolPromotionMaxTotal(
"arm-promote-constant-max-total", cl::Hidden, "arm-promote-constant-max-total", cl::Hidden,
cl::desc("Maximum size of ALL constants to promote into a constant pool"), cl::desc("Maximum size of ALL constants to promote into a constant pool"),
cl::init(128)); cl::init(128));
static cl::opt<unsigned> cl::opt<unsigned>
MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden, MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden,
cl::desc("Maximum interleave factor for MVE VLDn to generate."), cl::desc("Maximum interleave factor for MVE VLDn to generate."),
cl::init(2)); cl::init(2));
// The APCS parameter registers. // The APCS parameter registers.
static const MCPhysReg GPRArgRegs[] = { static const MCPhysReg GPRArgRegs[] = {
ARM::R0, ARM::R1, ARM::R2, ARM::R3 ARM::R0, ARM::R1, ARM::R2, ARM::R3
}; };
▲ Show 20 Lines • Show All 9,991 Lines • Show Last 20 Lines

llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp

Show All 22 Lines
#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsARM.h" #include "llvm/IR/IntrinsicsARM.h"
#include "llvm/IR/PatternMatch.h" #include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h" #include "llvm/IR/Type.h"
#include "llvm/MC/SubtargetFeature.h" #include "llvm/MC/SubtargetFeature.h"
#include "llvm/Support/Casting.h" #include "llvm/Support/Casting.h"
#include "llvm/Support/MachineValueType.h" #include "llvm/Support/MachineValueType.h"
#include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetMachine.h"
#include "ARMISelLowering.h"
dmgreenUnsubmitted
Done
ReplyInline Actions

Do you need to include this?

dmgreen: Do you need to include this?
anwelAuthorUnsubmitted
Done
ReplyInline Actions

Oops, no. Will remove for commit.

anwel: Oops, no. Will remove for commit.
#include "llvm/Transforms/InstCombine/InstCombiner.h" #include "llvm/Transforms/InstCombine/InstCombiner.h"
#include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopUtils.h" #include "llvm/Transforms/Utils/LoopUtils.h"
#include <algorithm> #include <algorithm>
#include <cassert> #include <cassert>
#include <cstdint> #include <cstdint>
#include <utility> #include <utility>
using namespace llvm; using namespace llvm;
#define DEBUG_TYPE "armtti" #define DEBUG_TYPE "armtti"
static cl::opt<bool> EnableMaskedLoadStores( static cl::opt<bool> EnableMaskedLoadStores(
"enable-arm-maskedldst", cl::Hidden, cl::init(true), "enable-arm-maskedldst", cl::Hidden, cl::init(true),
cl::desc("Enable the generation of masked loads and stores")); cl::desc("Enable the generation of masked loads and stores"));
static cl::opt<bool> DisableLowOverheadLoops( static cl::opt<bool> DisableLowOverheadLoops(
"disable-arm-loloops", cl::Hidden, cl::init(false), "disable-arm-loloops", cl::Hidden, cl::init(false),
cl::desc("Disable the generation of low-overhead loops")); cl::desc("Disable the generation of low-overhead loops"));
extern cl::opt<TailPredication::Mode> EnableTailPredication; extern cl::opt<TailPredication::Mode> EnableTailPredication;
extern cl::opt<bool> EnableMaskedGatherScatters; extern cl::opt<bool> EnableMaskedGatherScatters;
extern cl::opt<unsigned> MVEMaxSupportedInterleaveFactor;
/// Convert a vector load intrinsic into a simple llvm load instruction. /// Convert a vector load intrinsic into a simple llvm load instruction.
/// This is beneficial when the underlying object being addressed comes /// This is beneficial when the underlying object being addressed comes
/// from a constant, since we get constant-folding for free. /// from a constant, since we get constant-folding for free.
static Value *simplifyNeonVld1(const IntrinsicInst &II, unsigned MemAlign, static Value *simplifyNeonVld1(const IntrinsicInst &II, unsigned MemAlign,
InstCombiner::BuilderTy &Builder) { InstCombiner::BuilderTy &Builder) {
auto *IntrAlign = dyn_cast<ConstantInt>(II.getArgOperand(1)); auto *IntrAlign = dyn_cast<ConstantInt>(II.getArgOperand(1));
if (!IntrAlign) if (!IntrAlign)
▲ Show 20 Lines • Show All 1,534 Lines • ▼ Show 20 Lines if (IntReductionsDisabled) {
return false; return false;
} }
} }
// Next, check that all instructions can be tail-predicated. // Next, check that all instructions can be tail-predicated.
PredicatedScalarEvolution PSE = LAI->getPSE(); PredicatedScalarEvolution PSE = LAI->getPSE();
SmallVector<Instruction *, 16> LoadStores; SmallVector<Instruction *, 16> LoadStores;
int ICmpCount = 0; int ICmpCount = 0;
int Stride = 0;
for (BasicBlock *BB : L->blocks()) { for (BasicBlock *BB : L->blocks()) {
for (Instruction &I : BB->instructionsWithoutDebug()) { for (Instruction &I : BB->instructionsWithoutDebug()) {
if (isa<PHINode>(&I)) if (isa<PHINode>(&I))
continue; continue;
if (!canTailPredicateInstruction(I, ICmpCount)) { if (!canTailPredicateInstruction(I, ICmpCount)) {
LLVM_DEBUG(dbgs() << "Instruction not allowed: "; I.dump()); LLVM_DEBUG(dbgs() << "Instruction not allowed: "; I.dump());
return false; return false;
} }
Type *T = I.getType(); Type *T = I.getType();
if (T->isPointerTy()) if (T->isPointerTy())
T = T->getPointerElementType(); T = T->getPointerElementType();
if (T->getScalarSizeInBits() > 32) { if (T->getScalarSizeInBits() > 32) {
LLVM_DEBUG(dbgs() << "Unsupported Type: "; T->dump()); LLVM_DEBUG(dbgs() << "Unsupported Type: "; T->dump());
return false; return false;
} }
if (isa<StoreInst>(I) || isa<LoadInst>(I)) { if (isa<StoreInst>(I) || isa<LoadInst>(I)) {
Value *Ptr = isa<LoadInst>(I) ? I.getOperand(0) : I.getOperand(1); Value *Ptr = isa<LoadInst>(I) ? I.getOperand(0) : I.getOperand(1);
int64_t NextStride = getPtrStride(PSE, Ptr, L); int64_t NextStride = getPtrStride(PSE, Ptr, L);
if (NextStride == 1) {
// TODO: for now only allow consecutive strides of 1. We could support // TODO: for now only allow consecutive strides of 1. We could support
// other strides as long as it is uniform, but let's keep it simple for // other strides as long as it is uniform, but let's keep it simple
// now. // for now.
if (Stride == 0 && NextStride == 1) { continue;
Stride = NextStride; } else if (NextStride == -1 ||
dmgreenUnsubmitted
Done
ReplyInline Actions

A load or a store can be vld2 or a vst2, neither of which can be tail folded unfortunately.

dmgreen: A load or a store can be vld2 or a vst2, neither of which can be tail folded unfortunately.
anwelAuthorUnsubmitted
Done
ReplyInline Actions

Right, I forgot about the vstr2's. In that case we can never allow a stride of 2 here, as the only instructions that get us here are loads and stores.

anwel: Right, I forgot about the `vstr2`'s. In that case we can never allow a stride of 2 here, as the…
samparkerUnsubmitted
Done
ReplyInline Actions

else ifs?

samparker: else ifs?
anwelAuthorUnsubmitted
Done
ReplyInline Actions

Good point.

anwel: Good point.
(NextStride == 2 && MVEMaxSupportedInterleaveFactor >= 2) ||
(NextStride == 4 && MVEMaxSupportedInterleaveFactor >= 4)) {
LLVM_DEBUG(dbgs()
<< "Consecutive strides of 2 found, vld2/vstr2 can't "
"be tail-predicated\n.");
samparkerUnsubmitted
Done
ReplyInline Actions

I think this would be easier to read if this was organised in stride order and separating the gather/scatter from the consecutive accesses. So when !EnableMaskedGatherScatter, getPtrStride should only ever be 1, right? So I don't think we have to track the 'NextStride' business.

samparker: I think this would be easier to read if this was organised in stride order and separating the…
anwelAuthorUnsubmitted
Done
ReplyInline Actions

Good point. We may as well get rid of that.

anwel: Good point. We may as well get rid of that.
dmgreenUnsubmitted
Done
ReplyInline Actions

Can you change this condition to something like

if (NextStride == -1 || (NextStride == 2 && MVEMaxSupportedInterleaveFactor >= 2) || (NextStride == 4 && MVEMaxSupportedInterleaveFactor >= 4))

That should hopefully make it more futureproof, and specifically rule out reverse loads even if the vectorizer changes to support them.

dmgreen: Can you change this condition to something like if (NextStride == -1 || (NextStride == 2 &&…
anwelAuthorUnsubmitted
Done
ReplyInline Actions

Of course I can, that sounds like a good idea.

anwel: Of course I can, that sounds like a good idea.
return false;
// TODO: don't tail predicate if there is a reversed load?
} else if (EnableMaskedGatherScatters) {
// Gather/scatters do allow loading from arbitrary strides, at
// least if they are loop invariant.
// TODO: Loop variant strides should in theory work, too, but
// this requires further testing.
const SCEV *PtrScev =
replaceSymbolicStrideSCEV(PSE, llvm::ValueToValueMap(), Ptr);
if (auto AR = dyn_cast<SCEVAddRecExpr>(PtrScev)) {
const SCEV *Step = AR->getStepRecurrence(*PSE.getSE());
dmgreenUnsubmitted
Done
ReplyInline Actions

Perhaps

if (auto AR = dyn_cast<SCEVAddRecExpr>(PtrScev)) {
dmgreen: Perhaps if (auto AR = dyn_cast<SCEVAddRecExpr>(PtrScev)) {
anwelAuthorUnsubmitted
Done
ReplyInline Actions

Sure

anwel: Sure
if (PSE.getSE()->isLoopInvariant(Step, L))
continue; continue;
} }
if (Stride != NextStride) { }
LLVM_DEBUG(dbgs() << "Different strides found, can't " LLVM_DEBUG(dbgs() << "Bad stride found, can't "
"tail-predicate\n."); "tail-predicate\n.");
return false; return false;
} }
} }
} }
}
LLVM_DEBUG(dbgs() << "tail-predication: all instructions allowed!\n"); LLVM_DEBUG(dbgs() << "tail-predication: all instructions allowed!\n");
return true; return true;
} }
bool ARMTTIImpl::preferPredicateOverEpilogue(Loop *L, LoopInfo *LI, bool ARMTTIImpl::preferPredicateOverEpilogue(Loop *L, LoopInfo *LI,
ScalarEvolution &SE, ScalarEvolution &SE,
AssumptionCache &AC, AssumptionCache &AC,
▲ Show 20 Lines • Show All 139 Lines • Show Last 20 Lines

llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll

  • This file was added.
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -loop-vectorize -force-vector-width=4 -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -enable-arm-maskedgatscat -tail-predication=force-enabled -S %s -o - | FileCheck %s
define void @test_stride1_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) {
; CHECK-LABEL: @test_stride1_4i32(
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[INDEX]], i32 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0
; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw i32 [[TMP0]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 2
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[BROADCAST_SPLAT:%.*]], i32 0
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[TMP0]], i32 [[TMP4]])
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP3]], i32 0
; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>*
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP6]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef)
; CHECK-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP0]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i32 0
; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>*
; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP7]], <4 x i32>* [[TMP10]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]]
; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
;
entry:
br label %for.body
for.body: ; preds = %for.body.preheader, %for.body
%i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
%mul = mul nuw nsw i32 %i.023, 1
%add5 = add nuw nsw i32 %mul, 2
%arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
%0 = load i32, i32* %arrayidx6, align 4
%add7 = add nsw i32 5, %0
%arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
store i32 %add7, i32* %arrayidx9, align 4
%inc = add nuw nsw i32 %i.023, 1
%exitcond.not = icmp eq i32 %inc, %n
br i1 %exitcond.not, label %end, label %for.body
end: ; preds = %end, %entry
ret void
}
define void @test_stride-1_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) {
; CHECK-LABEL: @test_stride-1_4i32(
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ]
; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], 0
; CHECK-NEXT: [[TMP9:%.*]] = mul nuw nsw i32 [[TMP8]], -1
; CHECK-NEXT: [[TMP10:%.*]] = add nuw nsw i32 [[TMP9]], 2
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 0
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i32 -3
; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[TMP15:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[REVERSE]]
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP8]]
; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP16]], i32 0
; CHECK-NEXT: [[TMP18:%.*]] = bitcast i32* [[TMP17]] to <4 x i32>*
; CHECK-NEXT: store <4 x i32> [[TMP15]], <4 x i32>* [[TMP18]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]]
; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !4
;
entry:
br label %for.body
for.body: ; preds = %for.body.preheader, %for.body
%i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
%mul = mul nuw nsw i32 %i.023, -1
%add5 = add nuw nsw i32 %mul, 2
%arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
%0 = load i32, i32* %arrayidx6, align 4
%add7 = add nsw i32 5, %0
%arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
store i32 %add7, i32* %arrayidx9, align 4
%inc = add nuw nsw i32 %i.023, 1
%exitcond.not = icmp eq i32 %inc, %n
br i1 %exitcond.not, label %end, label %for.body
end: ; preds = %end, %entry
ret void
}
define void @test_stride2_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) {
;
; CHECK-LABEL: @test_stride2_4i32(
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ]
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 0
; CHECK-NEXT: [[TMP3:%.*]] = mul nuw nsw i32 [[TMP2]], 2
; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i32 [[TMP3]], 2
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i32 0
; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>*
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP7]], align 4
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP2]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0
; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>*
; CHECK-NEXT: store <4 x i32> [[TMP8]], <4 x i32>* [[TMP11]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]]
; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !6
;
entry:
br label %for.body
for.body: ; preds = %for.body.preheader, %for.body
%i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
%mul = mul nuw nsw i32 %i.023, 2
%add5 = add nuw nsw i32 %mul, 2
%arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
%0 = load i32, i32* %arrayidx6, align 4
%add7 = add nsw i32 5, %0
%arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
store i32 %add7, i32* %arrayidx9, align 4
%inc = add nuw nsw i32 %i.023, 1
%exitcond.not = icmp eq i32 %inc, %n
br i1 %exitcond.not, label %end, label %for.body
end: ; preds = %end, %entry
ret void
}
define void @test_stride3_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) {
; CHECK-LABEL: @test_stride3_4i32(
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3
; CHECK-NEXT: [[TMP4:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], <i32 3, i32 3, i32 3, i32 3>
; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw <4 x i32> [[TMP4]], <i32 2, i32 2, i32 2, i32 2>
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], <4 x i32> [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[BROADCAST_SPLAT:%.*]], i32 0
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[TMP0]], i32 [[TMP7]])
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP6]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_GATHER]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP0]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0
; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>*
; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP8]], <4 x i32>* [[TMP11]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]]
; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !8
;
entry:
br label %for.body
for.body: ; preds = %for.body.preheader, %for.body
%i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
%mul = mul nuw nsw i32 %i.023, 3
%add5 = add nuw nsw i32 %mul, 2
%arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
%0 = load i32, i32* %arrayidx6, align 4
%add7 = add nsw i32 5, %0
%arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
store i32 %add7, i32* %arrayidx9, align 4
%inc = add nuw nsw i32 %i.023, 1
%exitcond.not = icmp eq i32 %inc, %n
br i1 %exitcond.not, label %end, label %for.body
end: ; preds = %end, %entry
ret void
}
define void @test_stride4_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) {
; CHECK-LABEL: @test_stride4_4i32(
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3
; CHECK-NEXT: [[TMP4:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw <4 x i32> [[TMP4]], <i32 2, i32 2, i32 2, i32 2>
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], <4 x i32> [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[BROADCAST_SPLAT:%.*]], i32 0
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[TMP0]], i32 [[TMP7]])
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP6]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_GATHER]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP0]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0
; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>*
; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP8]], <4 x i32>* [[TMP11]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]]
; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP10:!llvm.loop !.*]]
;
entry:
br label %for.body
for.body: ; preds = %for.body.preheader, %for.body
%i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
%mul = mul nuw nsw i32 %i.023, 4
%add5 = add nuw nsw i32 %mul, 2
%arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
%0 = load i32, i32* %arrayidx6, align 4
%add7 = add nsw i32 5, %0
%arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
store i32 %add7, i32* %arrayidx9, align 4
%inc = add nuw nsw i32 %i.023, 1
%exitcond.not = icmp eq i32 %inc, %n
br i1 %exitcond.not, label %end, label %for.body
end: ; preds = %end, %entry
ret void
}
define void @test_stride_loopinvar_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n, i32 %stride) {
; CHECK-LABEL: @test_stride_loopinvar_4i32(
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[INDEX]], i32 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 0
; CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], [[STRIDE:%.*]]
; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], 2
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[BROADCAST_SPLAT]], i32 0
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[TMP1]], i32 [[TMP5]])
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0
; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP7]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0
; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>*
; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP8]], <4 x i32>* [[TMP11]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]]
; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP12:!llvm.loop !.*]]
;
entry:
br label %for.body
for.body: ; preds = %for.body.preheader, %for.body
%i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
%mul = mul nuw nsw i32 %i.023, %stride
%add5 = add nuw nsw i32 %mul, 2
%arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
%0 = load i32, i32* %arrayidx6, align 4
%add7 = add nsw i32 5, %0
%arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
store i32 %add7, i32* %arrayidx9, align 4
%inc = add nuw nsw i32 %i.023, 1
%exitcond.not = icmp eq i32 %inc, %n
br i1 %exitcond.not, label %end, label %for.body
end: ; preds = %end, %entry
ret void
}
define void @test_stride_noninvar_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) {
; CHECK-LABEL: @test_stride_noninvar_4i32(
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND2:%.*]] = phi <4 x i32> [ <i32 3, i32 11, i32 19, i32 27>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 0
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 3
; CHECK-NEXT: [[TMP5:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], [[VEC_IND2]]
; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw <4 x i32> [[TMP5]], <i32 2, i32 2, i32 2, i32 2>
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], <4 x i32> [[TMP6]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP7]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_GATHER]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0
; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>*
; CHECK-NEXT: store <4 x i32> [[TMP8]], <4 x i32>* [[TMP11]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
; CHECK-NEXT: [[VEC_IND_NEXT3]] = add <4 x i32> [[VEC_IND2]], <i32 32, i32 32, i32 32, i32 32>
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]]
; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP14:!llvm.loop !.*]]
;
entry:
br label %for.body
for.body: ; preds = %for.body.preheader, %for.body
%i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
%stride = phi i32 [ %next.stride, %for.body ], [ 3, %entry ]
%mul = mul nuw nsw i32 %i.023, %stride
%add5 = add nuw nsw i32 %mul, 2
%arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
%0 = load i32, i32* %arrayidx6, align 4
%add7 = add nsw i32 5, %0
%arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
store i32 %add7, i32* %arrayidx9, align 4
%inc = add nuw nsw i32 %i.023, 1
%next.stride = add nuw nsw i32 %stride, 8
%exitcond.not = icmp eq i32 %inc, %n
br i1 %exitcond.not, label %end, label %for.body
end: ; preds = %end, %entry
ret void
}
define void @test_stride_noninvar2_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n) {
; CHECK-LABEL: @test_stride_noninvar2_4i32(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
; CHECK-NEXT: [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-NEXT: [[STRIDE:%.*]] = phi i32 [ [[NEXT_STRIDE:%.*]], [[FOR_BODY]] ], [ 3, [[ENTRY]] ]
; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], [[STRIDE]]
; CHECK-NEXT: [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2
; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i32 [[ADD5]]
; CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4
; CHECK-NEXT: [[ADD7:%.*]] = add nsw i32 5, [[TMP0]]
; CHECK-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[I_023]]
; CHECK-NEXT: store i32 [[ADD7]], i32* [[ARRAYIDX9]], align 4
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_023]], 1
; CHECK-NEXT: [[NEXT_STRIDE]] = mul nuw nsw i32 [[STRIDE]], 8
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N:%.*]]
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[END:%.*]], label [[FOR_BODY]]
; CHECK: end:
; CHECK-NEXT: ret void
;
entry:
br label %for.body
for.body: ; preds = %for.body.preheader, %for.body
%i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
%stride = phi i32 [ %next.stride, %for.body ], [ 3, %entry ]
%mul = mul nuw nsw i32 %i.023, %stride
%add5 = add nuw nsw i32 %mul, 2
%arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
%0 = load i32, i32* %arrayidx6, align 4
%add7 = add nsw i32 5, %0
%arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
store i32 %add7, i32* %arrayidx9, align 4
%inc = add nuw nsw i32 %i.023, 1
%next.stride = mul nuw nsw i32 %stride, 8
%exitcond.not = icmp eq i32 %inc, %n
br i1 %exitcond.not, label %end, label %for.body
end: ; preds = %end, %entry
ret void
}
define void @test_stride_noninvar3_4i32(i32* readonly %data, i32* noalias nocapture %dst, i32 %n, i32 %x) {
; CHECK-LABEL: @test_stride_noninvar3_4i32(
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND4:%.*]] = phi <4 x i32> [ [[INDUCTION:%.*]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 0
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[INDEX]], 3
; CHECK-NEXT: [[TMP7:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], [[VEC_IND4]]
; CHECK-NEXT: [[TMP8:%.*]] = add nuw nsw <4 x i32> [[TMP7]], <i32 2, i32 2, i32 2, i32 2>
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], <4 x i32> [[TMP8]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP9]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> <i32 5, i32 5, i32 5, i32 5>, [[WIDE_MASKED_GATHER]]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[TMP3]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 0
; CHECK-NEXT: [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <4 x i32>*
; CHECK-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP13]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
; CHECK-NEXT: [[VEC_IND_NEXT5]] = add <4 x i32> [[VEC_IND4]], [[DOTSPLAT3:%.*]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]]
; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP16:!llvm.loop !.*]]
;
entry:
br label %for.body
for.body: ; preds = %for.body.preheader, %for.body
%i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
%stride = phi i32 [ %next.stride, %for.body ], [ 3, %entry ]
%mul = mul nuw nsw i32 %i.023, %stride
%add5 = add nuw nsw i32 %mul, 2
%arrayidx6 = getelementptr inbounds i32, i32* %data, i32 %add5
%0 = load i32, i32* %arrayidx6, align 4
%add7 = add nsw i32 5, %0
%arrayidx9 = getelementptr inbounds i32, i32* %dst, i32 %i.023
store i32 %add7, i32* %arrayidx9, align 4
%inc = add nuw nsw i32 %i.023, 1
%next.stride = add nuw nsw i32 %stride, %x
%exitcond.not = icmp eq i32 %inc, %n
br i1 %exitcond.not, label %end, label %for.body
end: ; preds = %end, %entry
ret void
}
declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>)
declare void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32>, <4 x i32*>, i32, <4 x i1>)