This is an archive of the discontinued LLVM Phabricator instance.

Differential D147976

[LSV] Improve chain splitting in some corner cases.
ClosedPublic

Authored by tra on Apr 10 2023, 4:00 PM.

Details

Reviewers
arsenm
jlebar
Commits
rGfaa631f939e4: [LSV] Improve chain splitting in some corner cases.
Summary

Currently we happen to split a chain of 12xi8 accesses into 6xi8 + 6xi8, which
produces rather suboptimal code.

This change attempts to split-off non-multiples of 4bytes at the end and if that
does not work, splits on the smaller power-of-2 boundary.

Diff Detail

Event Timeline

tra created this revision.Apr 10 2023, 4:00 PM
Herald added a project: Restricted Project. · View Herald TranscriptApr 10 2023, 4:00 PM
Herald added subscribers: mattd, gchakrabarti, asavonic and 2 others. · View Herald Transcript
tra published this revision for review.Apr 10 2023, 4:02 PM
tra added reviewers: arsenm, jlebar.
Herald added a project: Restricted Project. · View Herald TranscriptApr 10 2023, 4:02 PM
Herald added subscribers: llvm-commits, • pcwang-thead, wdng, jholewinski. · View Herald Transcript
Harbormaster completed remote builds in B224662: Diff 512279.Apr 10 2023, 4:41 PM
tra updated this revision to Diff 512510.Apr 11 2023, 10:33 AM

Minor cleanups.

jlebar accepted this revision.Apr 11 2023, 10:56 AM

Thanks, Art.

llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
672

nit: add {}?

This revision is now accepted and ready to land.Apr 11 2023, 10:56 AM
Harbormaster completed remote builds in B224817: Diff 512510.Apr 11 2023, 11:06 AM
tra updated this revision to Diff 512534.Apr 11 2023, 11:43 AM

rearrange comment.

tra updated this revision to Diff 512536.Apr 11 2023, 11:49 AM

Fixed alignment info in a test.

tra updated this revision to Diff 512546.Apr 11 2023, 12:14 PM

Added fully aligned int8x8a8 test.

Harbormaster completed remote builds in B224838: Diff 512546.Apr 11 2023, 1:08 PM
Closed by commit rGfaa631f939e4: [LSV] Improve chain splitting in some corner cases. (authored by tra). · Explain WhyApr 17 2023, 1:42 PM
This revision was automatically updated to reflect the committed changes.
tra added a commit: rGfaa631f939e4: [LSV] Improve chain splitting in some corner cases..

Revision Contents

PathSize
llvm/
lib/
Transforms/
Vectorize/
16 lines
test/
Transforms/
LoadStoreVectorizer/
NVPTX/
309 lines

Diff 514390

llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp

Show First 20 Lines • Show All 659 Lines • ▼ Show 20 Lines if (I->use_empty())
I->eraseFromParent(); I->eraseFromParent();
} }
std::pair<ArrayRef<Instruction *>, ArrayRef<Instruction *>> std::pair<ArrayRef<Instruction *>, ArrayRef<Instruction *>>
Vectorizer::splitOddVectorElts(ArrayRef<Instruction *> Chain, Vectorizer::splitOddVectorElts(ArrayRef<Instruction *> Chain,
unsigned ElementSizeBits) { unsigned ElementSizeBits) {
unsigned ElementSizeBytes = ElementSizeBits / 8; unsigned ElementSizeBytes = ElementSizeBits / 8;
unsigned SizeBytes = ElementSizeBytes * Chain.size(); unsigned SizeBytes = ElementSizeBytes * Chain.size();
unsigned NumLeft = (SizeBytes - (SizeBytes % 4)) / ElementSizeBytes; unsigned LeftBytes = (SizeBytes - (SizeBytes % 4));
if (NumLeft == Chain.size()) { // If we're already a multiple of 4 bytes or the whole chain is shorter than 4
if ((NumLeft & 1) == 0) // bytes, then try splitting down on power-of-2 boundary.
NumLeft /= 2; // Split even in half if (LeftBytes == SizeBytes || LeftBytes == 0)
else LeftBytes = PowerOf2Ceil(SizeBytes) / 2;
jlebarUnsubmitted
Not Done
ReplyInline Actions

nit: add {}?

jlebar: nit: add {}?
--NumLeft; // Split off last element unsigned NumLeft = LeftBytes / ElementSizeBytes;
} else if (NumLeft == 0) if (NumLeft == 0)
NumLeft = 1; NumLeft = 1;
LLVM_DEBUG(dbgs() << "LSV: Splitting the chain into " << NumLeft << "+"
<< Chain.size() - NumLeft << " elements\n");
return std::make_pair(Chain.slice(0, NumLeft), Chain.slice(NumLeft)); return std::make_pair(Chain.slice(0, NumLeft), Chain.slice(NumLeft));
} }
ArrayRef<Instruction *> ArrayRef<Instruction *>
Vectorizer::getVectorizablePrefix(ArrayRef<Instruction *> Chain) { Vectorizer::getVectorizablePrefix(ArrayRef<Instruction *> Chain) {
// These are in BB order, unlike Chain, which is in address order. // These are in BB order, unlike Chain, which is in address order.
SmallVector<Instruction *, 16> MemoryInstrs; SmallVector<Instruction *, 16> MemoryInstrs;
SmallVector<Instruction *, 16> ChainInstrs; SmallVector<Instruction *, 16> ChainInstrs;
▲ Show 20 Lines • Show All 661 Lines • Show Last 20 Lines

llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i8.ll

  • This file was added.
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; Vectorize and emit valid code (Issue #54896).
define void @int8x3a2(ptr nocapture align 2 %ptr) {
%ptr0 = getelementptr i8, ptr %ptr, i64 0
%ptr1 = getelementptr i8, ptr %ptr, i64 1
%ptr2 = getelementptr i8, ptr %ptr, i64 2
%l0 = load i8, ptr %ptr0, align 2
%l1 = load i8, ptr %ptr1, align 1
%l2 = load i8, ptr %ptr2, align 2
store i8 %l2, ptr %ptr0, align 2
store i8 %l1, ptr %ptr1, align 1
store i8 %l0, ptr %ptr2, align 2
ret void
; CHECK-LABEL: @int8x3a2
; CHECK-DAG: load <2 x i8>
; CHECK-DAG: load i8
; CHECK-DAG: store <2 x i8>
; CHECK-DAG: store i8
}
define void @int8x3a4(ptr nocapture align 4 %ptr) {
%ptr0 = getelementptr i8, ptr %ptr, i64 0
%ptr1 = getelementptr i8, ptr %ptr, i64 1
%ptr2 = getelementptr i8, ptr %ptr, i64 2
%l0 = load i8, ptr %ptr0, align 4
%l1 = load i8, ptr %ptr1, align 1
%l2 = load i8, ptr %ptr2, align 2
store i8 %l2, ptr %ptr0, align 2
store i8 %l1, ptr %ptr1, align 1
store i8 %l0, ptr %ptr2, align 4
ret void
; CHECK-LABEL: @int8x3a4
; CHECK: load <3 x i8>
; CHECK: store <3 x i8>
}
define void @int8x12a4(ptr nocapture align 4 %ptr) {
%ptr0 = getelementptr i8, ptr %ptr, i64 0
%ptr1 = getelementptr i8, ptr %ptr, i64 1
%ptr2 = getelementptr i8, ptr %ptr, i64 2
%ptr3 = getelementptr i8, ptr %ptr, i64 3
%ptr4 = getelementptr i8, ptr %ptr, i64 4
%ptr5 = getelementptr i8, ptr %ptr, i64 5
%ptr6 = getelementptr i8, ptr %ptr, i64 6
%ptr7 = getelementptr i8, ptr %ptr, i64 7
%ptr8 = getelementptr i8, ptr %ptr, i64 8
%ptr9 = getelementptr i8, ptr %ptr, i64 9
%ptra = getelementptr i8, ptr %ptr, i64 10
%ptrb = getelementptr i8, ptr %ptr, i64 11
%l0 = load i8, ptr %ptr0, align 4
%l1 = load i8, ptr %ptr1, align 1
%l2 = load i8, ptr %ptr2, align 2
%l3 = load i8, ptr %ptr3, align 1
%l4 = load i8, ptr %ptr4, align 4
%l5 = load i8, ptr %ptr5, align 1
%l6 = load i8, ptr %ptr6, align 2
%l7 = load i8, ptr %ptr7, align 1
%l8 = load i8, ptr %ptr8, align 4
%l9 = load i8, ptr %ptr9, align 1
%la = load i8, ptr %ptra, align 2
%lb = load i8, ptr %ptrb, align 1
store i8 %lb, ptr %ptr0, align 4
store i8 %la, ptr %ptr1, align 1
store i8 %l9, ptr %ptr2, align 2
store i8 %l8, ptr %ptr3, align 1
store i8 %l7, ptr %ptr4, align 4
store i8 %l6, ptr %ptr5, align 1
store i8 %l5, ptr %ptr6, align 2
store i8 %l4, ptr %ptr7, align 1
store i8 %l3, ptr %ptr8, align 4
store i8 %l2, ptr %ptr9, align 1
store i8 %l1, ptr %ptra, align 2
store i8 %l0, ptr %ptrb, align 1
ret void
; CHECK-LABEL: @int8x12a4
; CHECK: load <4 x i8>
; CHECK: load <4 x i8>
; CHECK: load <4 x i8>
; CHECK: store <4 x i8>
; CHECK: store <4 x i8>
; CHECK: store <4 x i8>
}
define void @int8x16a4(ptr nocapture align 4 %ptr) {
%ptr0 = getelementptr i8, ptr %ptr, i64 0
%ptr1 = getelementptr i8, ptr %ptr, i64 1
%ptr2 = getelementptr i8, ptr %ptr, i64 2
%ptr3 = getelementptr i8, ptr %ptr, i64 3
%ptr4 = getelementptr i8, ptr %ptr, i64 4
%ptr5 = getelementptr i8, ptr %ptr, i64 5
%ptr6 = getelementptr i8, ptr %ptr, i64 6
%ptr7 = getelementptr i8, ptr %ptr, i64 7
%ptr8 = getelementptr i8, ptr %ptr, i64 8
%ptr9 = getelementptr i8, ptr %ptr, i64 9
%ptra = getelementptr i8, ptr %ptr, i64 10
%ptrb = getelementptr i8, ptr %ptr, i64 11
%ptrc = getelementptr i8, ptr %ptr, i64 12
%ptrd = getelementptr i8, ptr %ptr, i64 13
%ptre = getelementptr i8, ptr %ptr, i64 14
%ptrf = getelementptr i8, ptr %ptr, i64 15
%l0 = load i8, ptr %ptr0, align 4
%l1 = load i8, ptr %ptr1, align 1
%l2 = load i8, ptr %ptr2, align 2
%l3 = load i8, ptr %ptr3, align 1
%l4 = load i8, ptr %ptr4, align 4
%l5 = load i8, ptr %ptr5, align 1
%l6 = load i8, ptr %ptr6, align 2
%l7 = load i8, ptr %ptr7, align 1
%l8 = load i8, ptr %ptr8, align 4
%l9 = load i8, ptr %ptr9, align 1
%la = load i8, ptr %ptra, align 2
%lb = load i8, ptr %ptrb, align 1
%lc = load i8, ptr %ptrc, align 4
%ld = load i8, ptr %ptrd, align 1
%le = load i8, ptr %ptre, align 2
%lf = load i8, ptr %ptrf, align 1
store i8 %lf, ptr %ptrc, align 4
store i8 %le, ptr %ptrd, align 1
store i8 %ld, ptr %ptre, align 2
store i8 %lc, ptr %ptrf, align 1
store i8 %lb, ptr %ptr0, align 4
store i8 %la, ptr %ptr1, align 1
store i8 %l9, ptr %ptr2, align 2
store i8 %l8, ptr %ptr3, align 1
store i8 %l7, ptr %ptr4, align 4
store i8 %l6, ptr %ptr5, align 1
store i8 %l5, ptr %ptr6, align 2
store i8 %l4, ptr %ptr7, align 1
store i8 %l3, ptr %ptr8, align 4
store i8 %l2, ptr %ptr9, align 1
store i8 %l1, ptr %ptra, align 2
store i8 %l0, ptr %ptrb, align 1
ret void
; CHECK-LABEL: @int8x16a4
; CHECK: load <4 x i8>
; CHECK: load <4 x i8>
; CHECK: load <4 x i8>
; CHECK: load <4 x i8>
; CHECK: store <4 x i8>
; CHECK: store <4 x i8>
; CHECK: store <4 x i8>
; CHECK: store <4 x i8>
}
define void @int8x8a8(ptr nocapture align 8 %ptr) {
%ptr0 = getelementptr i8, ptr %ptr, i64 0
%ptr1 = getelementptr i8, ptr %ptr, i64 1
%ptr2 = getelementptr i8, ptr %ptr, i64 2
%ptr3 = getelementptr i8, ptr %ptr, i64 3
%ptr4 = getelementptr i8, ptr %ptr, i64 4
%ptr5 = getelementptr i8, ptr %ptr, i64 5
%ptr6 = getelementptr i8, ptr %ptr, i64 6
%ptr7 = getelementptr i8, ptr %ptr, i64 7
%l0 = load i8, ptr %ptr0, align 8
%l1 = load i8, ptr %ptr1, align 1
%l2 = load i8, ptr %ptr2, align 2
%l3 = load i8, ptr %ptr3, align 1
%l4 = load i8, ptr %ptr4, align 4
%l5 = load i8, ptr %ptr5, align 1
%l6 = load i8, ptr %ptr6, align 2
%l7 = load i8, ptr %ptr7, align 1
store i8 %l7, ptr %ptr0, align 8
store i8 %l6, ptr %ptr1, align 1
store i8 %l5, ptr %ptr2, align 2
store i8 %l4, ptr %ptr3, align 1
store i8 %l3, ptr %ptr4, align 4
store i8 %l2, ptr %ptr5, align 1
store i8 %l1, ptr %ptr6, align 2
store i8 %l0, ptr %ptr7, align 1
ret void
; CHECK-LABEL: @int8x8a8
; CHECK: load <8 x i8>
; CHECK: store <8 x i8>
}
define void @int8x12a8(ptr nocapture align 8 %ptr) {
%ptr0 = getelementptr i8, ptr %ptr, i64 0
%ptr1 = getelementptr i8, ptr %ptr, i64 1
%ptr2 = getelementptr i8, ptr %ptr, i64 2
%ptr3 = getelementptr i8, ptr %ptr, i64 3
%ptr4 = getelementptr i8, ptr %ptr, i64 4
%ptr5 = getelementptr i8, ptr %ptr, i64 5
%ptr6 = getelementptr i8, ptr %ptr, i64 6
%ptr7 = getelementptr i8, ptr %ptr, i64 7
%ptr8 = getelementptr i8, ptr %ptr, i64 8
%ptr9 = getelementptr i8, ptr %ptr, i64 9
%ptra = getelementptr i8, ptr %ptr, i64 10
%ptrb = getelementptr i8, ptr %ptr, i64 11
%l0 = load i8, ptr %ptr0, align 8
%l1 = load i8, ptr %ptr1, align 1
%l2 = load i8, ptr %ptr2, align 2
%l3 = load i8, ptr %ptr3, align 1
%l4 = load i8, ptr %ptr4, align 4
%l5 = load i8, ptr %ptr5, align 1
%l6 = load i8, ptr %ptr6, align 2
%l7 = load i8, ptr %ptr7, align 1
%l8 = load i8, ptr %ptr8, align 8
%l9 = load i8, ptr %ptr9, align 1
%la = load i8, ptr %ptra, align 2
%lb = load i8, ptr %ptrb, align 1
store i8 %lb, ptr %ptr0, align 8
store i8 %la, ptr %ptr1, align 1
store i8 %l9, ptr %ptr2, align 2
store i8 %l8, ptr %ptr3, align 1
store i8 %l7, ptr %ptr4, align 4
store i8 %l6, ptr %ptr5, align 1
store i8 %l5, ptr %ptr6, align 2
store i8 %l4, ptr %ptr7, align 1
store i8 %l3, ptr %ptr8, align 8
store i8 %l2, ptr %ptr9, align 1
store i8 %l1, ptr %ptra, align 2
store i8 %l0, ptr %ptrb, align 1
ret void
; CHECK-LABEL: @int8x12a8
; CHECK-DAG: load <8 x i8>
; CHECK-DAG: load <4 x i8>
; CHECK-DAG: store <8 x i8>
; CHECK-DAG: store <4 x i8>
}
define void @int8x16a8(ptr nocapture align 8 %ptr) {
%ptr0 = getelementptr i8, ptr %ptr, i64 0
%ptr1 = getelementptr i8, ptr %ptr, i64 1
%ptr2 = getelementptr i8, ptr %ptr, i64 2
%ptr3 = getelementptr i8, ptr %ptr, i64 3
%ptr4 = getelementptr i8, ptr %ptr, i64 4
%ptr5 = getelementptr i8, ptr %ptr, i64 5
%ptr6 = getelementptr i8, ptr %ptr, i64 6
%ptr7 = getelementptr i8, ptr %ptr, i64 7
%ptr8 = getelementptr i8, ptr %ptr, i64 8
%ptr9 = getelementptr i8, ptr %ptr, i64 9
%ptra = getelementptr i8, ptr %ptr, i64 10
%ptrb = getelementptr i8, ptr %ptr, i64 11
%ptrc = getelementptr i8, ptr %ptr, i64 12
%ptrd = getelementptr i8, ptr %ptr, i64 13
%ptre = getelementptr i8, ptr %ptr, i64 14
%ptrf = getelementptr i8, ptr %ptr, i64 15
%l0 = load i8, ptr %ptr0, align 8
%l1 = load i8, ptr %ptr1, align 1
%l2 = load i8, ptr %ptr2, align 2
%l3 = load i8, ptr %ptr3, align 1
%l4 = load i8, ptr %ptr4, align 4
%l5 = load i8, ptr %ptr5, align 1
%l6 = load i8, ptr %ptr6, align 2
%l7 = load i8, ptr %ptr7, align 1
%l8 = load i8, ptr %ptr8, align 8
%l9 = load i8, ptr %ptr9, align 1
%la = load i8, ptr %ptra, align 2
%lb = load i8, ptr %ptrb, align 1
%lc = load i8, ptr %ptrc, align 4
%ld = load i8, ptr %ptrd, align 1
%le = load i8, ptr %ptre, align 2
%lf = load i8, ptr %ptrf, align 1
store i8 %lf, ptr %ptr0, align 8
store i8 %le, ptr %ptr1, align 1
store i8 %ld, ptr %ptr2, align 2
store i8 %lc, ptr %ptr3, align 1
store i8 %lb, ptr %ptr4, align 4
store i8 %la, ptr %ptr5, align 1
store i8 %l9, ptr %ptr6, align 2
store i8 %l8, ptr %ptr7, align 1
store i8 %l7, ptr %ptr8, align 8
store i8 %l6, ptr %ptr9, align 1
store i8 %l5, ptr %ptra, align 2
store i8 %l4, ptr %ptrb, align 1
store i8 %l3, ptr %ptrc, align 4
store i8 %l2, ptr %ptrd, align 1
store i8 %l1, ptr %ptre, align 2
store i8 %l0, ptr %ptrf, align 1
ret void
; CHECK-LABEL: @int8x16a8
; CHECK: load <8 x i8>
; CHECK: load <8 x i8>
; CHECK: store <8 x i8>
; CHECK: store <8 x i8>
}