This is an archive of the discontinued LLVM Phabricator instance.

Differential D147000

[RISCV] Cost model for general case of single vector permute
ClosedPublic

Authored by reames on Mar 27 2023, 1:46 PM.

Details

Reviewers
luke
craig.topper
asb
Commits
rG64f69e453ed3: [RISCV] Cost model for general case of single vector permute
Summary

The cost model was not accounting for the fact that we can generate vrgather + an index expression.

One thing to call out - I did not model the difference between vrgather and vrgatherei16. The result is the constant pool cost can be slightly understated on RV32. I'm not sure if we care, if we do, I can add in the handling for that case.

Diff Detail

Event Timeline

reames created this revision.Mar 27 2023, 1:46 PM
Herald added a project: Restricted Project. · View Herald TranscriptMar 27 2023, 1:46 PM
Herald added subscribers: jobnoorman, VincentWu, vkmr and 29 others. · View Herald Transcript
reames requested review of this revision.Mar 27 2023, 1:46 PM
Herald added a project: Restricted Project. · View Herald TranscriptMar 27 2023, 1:46 PM
Herald added subscribers: • pcwang-thead, eopXD, MaskRay. · View Herald Transcript
Harbormaster completed remote builds in B222105: Diff 508785.Mar 27 2023, 3:02 PM
luke accepted this revision.Mar 28 2023, 3:22 AM

LGTM, this matches up with what I'm seeing generating random masks for shuffles.

Just a small observation: Could we improve codegen and use vrgatherei16 for i8 and n > 256? The generated code at the moment for a v512i8 vector shuffle is less than ideal

v512:                                   # @v512
	.cfi_startproc
# %bb.0:
	lui	a0, %hi(.LCPI1_0)
	ld	a0, %lo(.LCPI1_0)(a0)
	vsetivli	zero, 8, e64, m1, ta, ma
	vmv.s.x	v12, a0
	vsetivli	zero, 2, e64, m1, tu, ma
	vmv1r.v	v0, v12
	vslideup.vi	v0, v12, 1
	vsetivli	zero, 3, e64, m1, tu, ma
	vslideup.vi	v0, v12, 2
	vsetivli	zero, 4, e64, m1, tu, ma
	vslideup.vi	v0, v12, 3
	vsetivli	zero, 5, e64, m1, tu, ma
	vslideup.vi	v0, v12, 4
	vsetivli	zero, 6, e64, m1, tu, ma
	vslideup.vi	v0, v12, 5
	vsetivli	zero, 7, e64, m1, tu, ma
	vslideup.vi	v0, v12, 6
	vsetivli	zero, 8, e64, m1, ta, ma
	vslideup.vi	v0, v12, 7
	vsetivli	zero, 1, e8, m4, ta, ma
	vslidedown.vi	v12, v8, 1
	vmv.x.s	a1, v12
	li	a0, 512
	vsetvli	zero, a0, e8, m4, ta, ma
	vmv.v.x	v12, a1
	vsetivli	zero, 1, e8, m4, ta, ma
	vslidedown.vi	v16, v8, 5
	vmv.x.s	a1, v16
	lui	a2, %hi(.LCPI1_1)
	ld	a2, %lo(.LCPI1_1)(a2)
	vsetvli	zero, a0, e8, m4, ta, ma
	vmerge.vxm	v12, v12, a1, v0
	vsetivli	zero, 8, e64, m1, ta, ma
	vmv.s.x	v16, a2
	vsetivli	zero, 2, e64, m1, tu, ma
	vmv1r.v	v0, v16
	vslideup.vi	v0, v16, 1
	vsetivli	zero, 3, e64, m1, tu, ma
	vslideup.vi	v0, v16, 2
	vsetivli	zero, 4, e64, m1, tu, ma
	vslideup.vi	v0, v16, 3
	vsetivli	zero, 5, e64, m1, tu, ma
	vslideup.vi	v0, v16, 4
	vsetivli	zero, 6, e64, m1, tu, ma
	vslideup.vi	v0, v16, 5
	vsetivli	zero, 7, e64, m1, tu, ma
	vslideup.vi	v0, v16, 6
	vsetivli	zero, 8, e64, m1, ta, ma
	vslideup.vi	v0, v16, 7
	vsetivli	zero, 1, e8, m4, ta, ma
	vslidedown.vi	v16, v8, 4
	vmv.x.s	a1, v16
	lui	a2, %hi(.LCPI1_2)
	ld	a2, %lo(.LCPI1_2)(a2)
	vsetvli	zero, a0, e8, m4, ta, ma
	vmerge.vxm	v12, v12, a1, v0
	vsetivli	zero, 8, e64, m1, ta, ma
	vmv.s.x	v16, a2
	vsetivli	zero, 2, e64, m1, tu, ma
	vmv1r.v	v0, v16
	vslideup.vi	v0, v16, 1
	vsetivli	zero, 3, e64, m1, tu, ma
	vslideup.vi	v0, v16, 2
	vsetivli	zero, 4, e64, m1, tu, ma
	vslideup.vi	v0, v16, 3
	vsetivli	zero, 5, e64, m1, tu, ma
	vslideup.vi	v0, v16, 4
	vsetivli	zero, 6, e64, m1, tu, ma
	vslideup.vi	v0, v16, 5
	vsetivli	zero, 7, e64, m1, tu, ma
	vslideup.vi	v0, v16, 6
	vsetivli	zero, 8, e64, m1, ta, ma
	vslideup.vi	v0, v16, 7
	vsetivli	zero, 1, e8, m4, ta, ma
	vslidedown.vi	v16, v8, 7
	vmv.x.s	a1, v16
	lui	a2, %hi(.LCPI1_3)
	ld	a2, %lo(.LCPI1_3)(a2)
	vsetvli	zero, a0, e8, m4, ta, ma
	vmerge.vxm	v12, v12, a1, v0
	vsetivli	zero, 8, e64, m1, ta, ma
	vmv.s.x	v16, a2
	vsetivli	zero, 2, e64, m1, tu, ma
	vmv1r.v	v0, v16
	vslideup.vi	v0, v16, 1
	vsetivli	zero, 3, e64, m1, tu, ma
	vslideup.vi	v0, v16, 2
	vsetivli	zero, 4, e64, m1, tu, ma
	vslideup.vi	v0, v16, 3
	vsetivli	zero, 5, e64, m1, tu, ma
	vslideup.vi	v0, v16, 4
	vsetivli	zero, 6, e64, m1, tu, ma
	vslideup.vi	v0, v16, 5
	vsetivli	zero, 7, e64, m1, tu, ma
	vslideup.vi	v0, v16, 6
	vsetivli	zero, 8, e64, m1, ta, ma
	vslideup.vi	v0, v16, 7
	vsetivli	zero, 1, e8, m4, ta, ma
	vslidedown.vi	v16, v8, 3
	vmv.x.s	a1, v16
	lui	a2, %hi(.LCPI1_4)
	ld	a2, %lo(.LCPI1_4)(a2)
	vsetvli	zero, a0, e8, m4, ta, ma
	vmerge.vxm	v12, v12, a1, v0
	vsetivli	zero, 8, e64, m1, ta, ma
	vmv.s.x	v16, a2
	vsetivli	zero, 2, e64, m1, tu, ma
	vmv1r.v	v0, v16
	vslideup.vi	v0, v16, 1
	vsetivli	zero, 3, e64, m1, tu, ma
	vslideup.vi	v0, v16, 2
	vsetivli	zero, 4, e64, m1, tu, ma
	vslideup.vi	v0, v16, 3
	vsetivli	zero, 5, e64, m1, tu, ma
	vslideup.vi	v0, v16, 4
	vsetivli	zero, 6, e64, m1, tu, ma
	vslideup.vi	v0, v16, 5
	vsetivli	zero, 7, e64, m1, tu, ma
	vslideup.vi	v0, v16, 6
	vsetivli	zero, 8, e64, m1, ta, ma
	vslideup.vi	v0, v16, 7
	vsetivli	zero, 0, e8, m4, ta, ma
	vmv.x.s	a1, v8
	lui	a2, %hi(.LCPI1_5)
	ld	a2, %lo(.LCPI1_5)(a2)
	vsetvli	zero, a0, e8, m4, ta, ma
	vmerge.vxm	v12, v12, a1, v0
	vsetivli	zero, 8, e64, m1, ta, ma
	vmv.s.x	v16, a2
	vsetivli	zero, 2, e64, m1, tu, ma
	vmv1r.v	v0, v16
	vslideup.vi	v0, v16, 1
	vsetivli	zero, 3, e64, m1, tu, ma
	vslideup.vi	v0, v16, 2
	vsetivli	zero, 4, e64, m1, tu, ma
	vslideup.vi	v0, v16, 3
	vsetivli	zero, 5, e64, m1, tu, ma
	vslideup.vi	v0, v16, 4
	vsetivli	zero, 6, e64, m1, tu, ma
	vslideup.vi	v0, v16, 5
	vsetivli	zero, 7, e64, m1, tu, ma
	vslideup.vi	v0, v16, 6
	vsetivli	zero, 8, e64, m1, ta, ma
	vslideup.vi	v0, v16, 7
	vsetivli	zero, 1, e8, m4, ta, ma
	vslidedown.vi	v8, v8, 2
	vmv.x.s	a1, v8
	vsetvli	zero, a0, e8, m4, ta, ma
	vmerge.vxm	v8, v12, a1, v0
	ret

Below is v256i8 for comparison, as we are modelling:

v256:                                   # @v256
	.cfi_startproc
# %bb.0:
	lui	a0, %hi(.LCPI0_0)
	addi	a0, a0, %lo(.LCPI0_0)
	li	a1, 32
	vsetvli	zero, a1, e64, m2, ta, ma
	vlse64.v	v12, (a0), zero
	li	a0, 256
	vsetvli	zero, a0, e8, m2, ta, ma
	vrgather.vv	v10, v8, v12
	vmv.v.v	v8, v10
	ret
This revision is now accepted and ready to land.Mar 28 2023, 3:22 AM
This revision was landed with ongoing or failed builds.Mar 28 2023, 7:34 AM
Closed by commit rG64f69e453ed3: [RISCV] Cost model for general case of single vector permute (authored by reames). · Explain Why
This revision was automatically updated to reflect the committed changes.
reames added a commit: rG64f69e453ed3: [RISCV] Cost model for general case of single vector permute.

Revision Contents

PathSize
llvm/
lib/
Target/
RISCV/
13 lines
test/
Analysis/
CostModel/
RISCV/
17 lines
Transforms/
LoopVectorize/
RISCV/
46 lines

Diff 509006

llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp

Show First 20 Lines • Show All 274 Lines • ▼ Show 20 Lines case TTI::SK_PermuteSingleSrc: {
if (Mask[0] == 0 || Mask[0] == 1) { if (Mask[0] == 0 || Mask[0] == 1) {
auto DeinterleaveMask = createStrideMask(Mask[0], 2, Mask.size()); auto DeinterleaveMask = createStrideMask(Mask[0], 2, Mask.size());
// Example sequence: // Example sequence:
// vnsrl.wi v10, v8, 0 // vnsrl.wi v10, v8, 0
if (equal(DeinterleaveMask, Mask)) if (equal(DeinterleaveMask, Mask))
return LT.first * getLMULCost(LT.second); return LT.first * getLMULCost(LT.second);
} }
} }
// vrgather + cost of generating the mask constant.
// We model this for an unknown mask with a single vrgather.
if (LT.first == 1 &&
(LT.second.getScalarSizeInBits() != 8 ||
LT.second.getVectorNumElements() <= 256)) {
VectorType *IdxTy = VectorType::get(IntegerType::getInt8Ty(Tp->getContext()),
Tp->getElementCount());
InstructionCost IndexCost =
2 + getMemoryOpCost(Instruction::Load, IdxTy, DL.getABITypeAlign(IdxTy),
/*AddressSpace=*/0, CostKind);
return IndexCost + getLMULCost(LT.second);
}
} }
} }
} }
}; };
// Handle scalable vectors (and fixed vectors legalized to scalable vectors). // Handle scalable vectors (and fixed vectors legalized to scalable vectors).
switch (Kind) { switch (Kind) {
default: default:
▲ Show 20 Lines • Show All 1,300 Lines • Show Last 20 Lines

llvm/test/Analysis/CostModel/RISCV/shuffle-interleave.ll

; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 2 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 2
; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv32 -mattr=+v | FileCheck %s -check-prefixes=CHECK,RV32 ; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv32 -mattr=+v | FileCheck %s
; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv64 -mattr=+v | FileCheck %s -check-prefixes=CHECK,RV64 ; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv64 -mattr=+v | FileCheck %s
; The mask here interleaves (%v1, %v0), not (%v0, %v1): it should still be cheap. ; The mask here interleaves (%v1, %v0), not (%v0, %v1): it should still be cheap.
define <4 x i8> @interleave2_v2i8(<2 x i8> %v0, <2 x i8> %v1) { define <4 x i8> @interleave2_v2i8(<2 x i8> %v0, <2 x i8> %v1) {
; CHECK-LABEL: 'interleave2_v2i8' ; CHECK-LABEL: 'interleave2_v2i8'
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %concat = shufflevector <2 x i8> %v0, <2 x i8> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3> ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %concat = shufflevector <2 x i8> %v0, <2 x i8> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %res = shufflevector <4 x i8> %concat, <4 x i8> poison, <4 x i32> <i32 2, i32 0, i32 3, i32 1> ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %res = shufflevector <4 x i8> %concat, <4 x i8> poison, <4 x i32> <i32 2, i32 0, i32 3, i32 1>
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i8> %res ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i8> %res
; ;
Show All 21 Lines
; ;
%concat = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> %concat = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%res = shufflevector <8 x i32> %concat, <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7> %res = shufflevector <8 x i32> %concat, <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
ret <8 x i32> %res ret <8 x i32> %res
} }
; Should be expensive on RV32 because it can't widen ; Should be expensive on RV32 because it can't widen
define <8 x i64> @interleave2_v8i64(<4 x i64> %v0, <4 x i64> %v1) { define <8 x i64> @interleave2_v8i64(<4 x i64> %v0, <4 x i64> %v1) {
; RV32-LABEL: 'interleave2_v8i64' ; CHECK-LABEL: 'interleave2_v8i64'
; RV32-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %concat = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> ; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %concat = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; RV32-NEXT: Cost Model: Found an estimated cost of 70 for instruction: %res = shufflevector <8 x i64> %concat, <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7> ; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %res = shufflevector <8 x i64> %concat, <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
; RV32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %res ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %res
;
; RV64-LABEL: 'interleave2_v8i64'
; RV64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %concat = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; RV64-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %res = shufflevector <8 x i64> %concat, <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
; RV64-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %res
; ;
%concat = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> %concat = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%res = shufflevector <8 x i64> %concat, <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7> %res = shufflevector <8 x i64> %concat, <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
ret <8 x i64> %res ret <8 x i64> %res
} }
; TODO: getInstructionCost doesn't call getShuffleCost here because the shuffle changes length ; TODO: getInstructionCost doesn't call getShuffleCost here because the shuffle changes length
define {<4 x i8>, <4 x i8>} @deinterleave_2(<8 x i8> %v) { define {<4 x i8>, <4 x i8>} @deinterleave_2(<8 x i8> %v) {
Show All 13 Lines

llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll

Show First 20 Lines • Show All 72 Lines • ▼ Show 20 Linesloop:
br i1 %done, label %exit, label %loop br i1 %done, label %exit, label %loop
exit: exit:
ret void ret void
} }
define void @load_store_factor2_i64(ptr %p) { define void @load_store_factor2_i64(ptr %p) {
; CHECK-LABEL: @load_store_factor2_i64( ; CHECK-LABEL: @load_store_factor2_i64(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[TMP2]], i32 0
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP3]], align 4
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], <i64 1, i64 1, i64 1, i64 1>
; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP1]], 1
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i64> [[STRIDED_VEC1]], <i64 2, i64 2, i64 2, i64 2>
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[TMP6]], i32 -1
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP8]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, 1024
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop: ; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[NEXTI:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[OFFSET0:%.*]] = shl i64 [[I]], 1 ; CHECK-NEXT: [[OFFSET0:%.*]] = shl i64 [[I]], 1
; CHECK-NEXT: [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]] ; CHECK-NEXT: [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
; CHECK-NEXT: [[X0:%.*]] = load i64, ptr [[Q0]], align 4 ; CHECK-NEXT: [[X0:%.*]] = load i64, ptr [[Q0]], align 4
; CHECK-NEXT: [[Y0:%.*]] = add i64 [[X0]], 1 ; CHECK-NEXT: [[Y0:%.*]] = add i64 [[X0]], 1
; CHECK-NEXT: store i64 [[Y0]], ptr [[Q0]], align 4 ; CHECK-NEXT: store i64 [[Y0]], ptr [[Q0]], align 4
; CHECK-NEXT: [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1 ; CHECK-NEXT: [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
; CHECK-NEXT: [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]] ; CHECK-NEXT: [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
; CHECK-NEXT: [[X1:%.*]] = load i64, ptr [[Q1]], align 4 ; CHECK-NEXT: [[X1:%.*]] = load i64, ptr [[Q1]], align 4
; CHECK-NEXT: [[Y1:%.*]] = add i64 [[X1]], 2 ; CHECK-NEXT: [[Y1:%.*]] = add i64 [[X1]], 2
; CHECK-NEXT: store i64 [[Y1]], ptr [[Q1]], align 4 ; CHECK-NEXT: store i64 [[Y1]], ptr [[Q1]], align 4
; CHECK-NEXT: [[NEXTI]] = add i64 [[I]], 1 ; CHECK-NEXT: [[NEXTI]] = add i64 [[I]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024 ; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
; CHECK-NEXT: br i1 [[DONE]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK-NEXT: br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: exit: ; CHECK: exit:
; CHECK-NEXT: ret void ; CHECK-NEXT: ret void
; ;
entry: entry:
br label %loop br label %loop
loop: loop:
%i = phi i64 [0, %entry], [%nexti, %loop] %i = phi i64 [0, %entry], [%nexti, %loop]
▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines
; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP17]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> poison) ; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP17]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP18:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER2]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer) ; CHECK-NEXT: [[TMP18:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER2]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP18]], <vscale x 4 x ptr> [[TMP17]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP18]], <vscale x 4 x ptr> [[TMP17]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
; CHECK-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 4 ; CHECK-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP20]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP20]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: middle.block: ; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]] ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph: ; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop: ; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
Show All 9 Lines
; CHECK-NEXT: store i32 [[Y1]], ptr [[Q1]], align 4 ; CHECK-NEXT: store i32 [[Y1]], ptr [[Q1]], align 4
; CHECK-NEXT: [[OFFSET2:%.*]] = add i64 [[OFFSET1]], 1 ; CHECK-NEXT: [[OFFSET2:%.*]] = add i64 [[OFFSET1]], 1
; CHECK-NEXT: [[Q2:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET2]] ; CHECK-NEXT: [[Q2:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET2]]
; CHECK-NEXT: [[X2:%.*]] = load i32, ptr [[Q2]], align 4 ; CHECK-NEXT: [[X2:%.*]] = load i32, ptr [[Q2]], align 4
; CHECK-NEXT: [[Y2:%.*]] = add i32 [[X2]], 3 ; CHECK-NEXT: [[Y2:%.*]] = add i32 [[X2]], 3
; CHECK-NEXT: store i32 [[Y2]], ptr [[Q2]], align 4 ; CHECK-NEXT: store i32 [[Y2]], ptr [[Q2]], align 4
; CHECK-NEXT: [[NEXTI]] = add i64 [[I]], 1 ; CHECK-NEXT: [[NEXTI]] = add i64 [[I]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024 ; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
; CHECK-NEXT: br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK-NEXT: br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: exit: ; CHECK: exit:
; CHECK-NEXT: ret void ; CHECK-NEXT: ret void
; ;
entry: entry:
br label %loop br label %loop
loop: loop:
%i = phi i64 [0, %entry], [%nexti, %loop] %i = phi i64 [0, %entry], [%nexti, %loop]
▲ Show 20 Lines • Show All 218 Lines • ▼ Show 20 Lines
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[TMP0]] ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[Q]], i64 [[TMP1]] ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[Q]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[TMP10]], i32 0 ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[TMP10]], i32 0
; CHECK-NEXT: store <8 x i32> [[TMP8]], ptr [[TMP12]], align 4 ; CHECK-NEXT: store <8 x i32> [[TMP8]], ptr [[TMP12]], align 4
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[TMP10]], i32 8 ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[TMP10]], i32 8
; CHECK-NEXT: store <8 x i32> [[TMP9]], ptr [[TMP13]], align 4 ; CHECK-NEXT: store <8 x i32> [[TMP9]], ptr [[TMP13]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: middle.block: ; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, 1024 ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, 1024
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph: ; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop: ; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[OFFSET0:%.*]] = shl i64 [[I]], 1 ; CHECK-NEXT: [[OFFSET0:%.*]] = shl i64 [[I]], 1
; CHECK-NEXT: [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]] ; CHECK-NEXT: [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
; CHECK-NEXT: [[X0:%.*]] = load i32, ptr [[Q0]], align 4 ; CHECK-NEXT: [[X0:%.*]] = load i32, ptr [[Q0]], align 4
; CHECK-NEXT: [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1 ; CHECK-NEXT: [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
; CHECK-NEXT: [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]] ; CHECK-NEXT: [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET1]]
; CHECK-NEXT: [[X1:%.*]] = load i32, ptr [[Q1]], align 4 ; CHECK-NEXT: [[X1:%.*]] = load i32, ptr [[Q1]], align 4
; CHECK-NEXT: [[RES:%.*]] = add i32 [[X0]], [[X1]] ; CHECK-NEXT: [[RES:%.*]] = add i32 [[X0]], [[X1]]
; CHECK-NEXT: [[DST:%.*]] = getelementptr i32, ptr [[Q]], i64 [[I]] ; CHECK-NEXT: [[DST:%.*]] = getelementptr i32, ptr [[Q]], i64 [[I]]
; CHECK-NEXT: store i32 [[RES]], ptr [[DST]], align 4 ; CHECK-NEXT: store i32 [[RES]], ptr [[DST]], align 4
; CHECK-NEXT: [[NEXTI]] = add i64 [[I]], 1 ; CHECK-NEXT: [[NEXTI]] = add i64 [[I]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024 ; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
; CHECK-NEXT: br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK-NEXT: br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: exit: ; CHECK: exit:
; CHECK-NEXT: ret void ; CHECK-NEXT: ret void
; ;
entry: entry:
br label %loop br label %loop
loop: loop:
%i = phi i64 [0, %entry], [%nexti, %loop] %i = phi i64 [0, %entry], [%nexti, %loop]
Show All 33 Lines
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7> ; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], [[STRIDED_VEC1]] ; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], [[STRIDED_VEC1]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[TMP0]] ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0
; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP6]], align 4 ; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP6]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: middle.block: ; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, 1024 ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, 1024
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph: ; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop: ; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[OFFSET0:%.*]] = shl i64 [[I]], 1 ; CHECK-NEXT: [[OFFSET0:%.*]] = shl i64 [[I]], 1
; CHECK-NEXT: [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]] ; CHECK-NEXT: [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
; CHECK-NEXT: [[X0:%.*]] = load i64, ptr [[Q0]], align 4 ; CHECK-NEXT: [[X0:%.*]] = load i64, ptr [[Q0]], align 4
; CHECK-NEXT: [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1 ; CHECK-NEXT: [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1
; CHECK-NEXT: [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]] ; CHECK-NEXT: [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]]
; CHECK-NEXT: [[X1:%.*]] = load i64, ptr [[Q1]], align 4 ; CHECK-NEXT: [[X1:%.*]] = load i64, ptr [[Q1]], align 4
; CHECK-NEXT: [[RES:%.*]] = add i64 [[X0]], [[X1]] ; CHECK-NEXT: [[RES:%.*]] = add i64 [[X0]], [[X1]]
; CHECK-NEXT: [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I]] ; CHECK-NEXT: [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I]]
; CHECK-NEXT: store i64 [[RES]], ptr [[DST]], align 4 ; CHECK-NEXT: store i64 [[RES]], ptr [[DST]], align 4
; CHECK-NEXT: [[NEXTI]] = add i64 [[I]], 1 ; CHECK-NEXT: [[NEXTI]] = add i64 [[I]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024 ; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
; CHECK-NEXT: br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]] ; CHECK-NEXT: br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: exit: ; CHECK: exit:
; CHECK-NEXT: ret void ; CHECK-NEXT: ret void
; ;
entry: entry:
br label %loop br label %loop
loop: loop:
%i = phi i64 [0, %entry], [%nexti, %loop] %i = phi i64 [0, %entry], [%nexti, %loop]
Show All 20 Lines