This is an archive of the discontinued LLVM Phabricator instance.

Differential D125086

[VectorCombine] Attempt to fold select shuffles from reductions
ClosedPublic

Authored by dmgreen on May 6 2022, 3:08 AM.

Details

Reviewers
samtebbs
spatel
RKSimon
SjoerdMeijer
labrinea
Commits
rG6f9e1ea0efb9: [VectorCombine] Attempt to fold select shuffles from reductions
Summary

Given a commutative reduction leading from a shuffle, the order of the lanes on the shuffle are not important for the result. This means we can reorder the shuffle to something simpler, which it tries shuffling the first vector lanes first. This was D123494.

The new shuffle may not be profitable though, and if it is not we can try the folding of select shuffles from D123911. This, with some adjustment as the output lane ordering is now unimportant, can allow the final shuffle to simplify given the inputs to the patterns from D123911. Where as each transformation on their own are not profitable, to combination is.

We can only support a single shuffle when called from reductions, but we are able to sort the ReconstructMask, potentially allowing it to simplify to an identity or concat mask.

Diff Detail

Event Timeline

dmgreen created this revision.May 6 2022, 3:08 AM
Herald added a project: Restricted Project. · View Herald TranscriptMay 6 2022, 3:08 AM
Herald added a subscriber: hiraditya. · View Herald Transcript
dmgreen requested review of this revision.May 6 2022, 3:08 AM
Herald added a project: Restricted Project. · View Herald TranscriptMay 6 2022, 3:08 AM
Harbormaster completed remote builds in B163102: Diff 427353.May 6 2022, 4:41 AM
samtebbs accepted this revision.May 6 2022, 7:53 AM

Looks sensible to me, thanks

This revision is now accepted and ready to land.May 6 2022, 7:53 AM
This revision was landed with ongoing or failed builds.May 8 2022, 2:33 AM
Closed by commit rG6f9e1ea0efb9: [VectorCombine] Attempt to fold select shuffles from reductions (authored by dmgreen). · Explain Why
This revision was automatically updated to reflect the committed changes.
dmgreen added a commit: rG6f9e1ea0efb9: [VectorCombine] Attempt to fold select shuffles from reductions.

Revision Contents

PathSize
llvm/
lib/
Transforms/
Vectorize/
16 lines
test/
Transforms/
VectorCombine/
AArch64/
88 lines

Diff 427920

llvm/lib/Transforms/Vectorize/VectorCombine.cpp

Show First 20 Lines • Show All 98 Lines • ▼ Show 20 Linesprivate:
bool foldExtractExtract(Instruction &I); bool foldExtractExtract(Instruction &I);
bool foldBitcastShuf(Instruction &I); bool foldBitcastShuf(Instruction &I);
bool scalarizeBinopOrCmp(Instruction &I); bool scalarizeBinopOrCmp(Instruction &I);
bool foldExtractedCmps(Instruction &I); bool foldExtractedCmps(Instruction &I);
bool foldSingleElementStore(Instruction &I); bool foldSingleElementStore(Instruction &I);
bool scalarizeLoadExtract(Instruction &I); bool scalarizeLoadExtract(Instruction &I);
bool foldShuffleOfBinops(Instruction &I); bool foldShuffleOfBinops(Instruction &I);
bool foldShuffleFromReductions(Instruction &I); bool foldShuffleFromReductions(Instruction &I);
bool foldSelectShuffle(Instruction &I); bool foldSelectShuffle(Instruction &I, bool FromReduction = false);
void replaceValue(Value &Old, Value &New) { void replaceValue(Value &Old, Value &New) {
Old.replaceAllUsesWith(&New); Old.replaceAllUsesWith(&New);
if (auto *NewI = dyn_cast<Instruction>(&New)) { if (auto *NewI = dyn_cast<Instruction>(&New)) {
New.takeName(&Old); New.takeName(&Old);
Worklist.pushUsersToWorkList(*NewI); Worklist.pushUsersToWorkList(*NewI);
Worklist.pushValue(NewI); Worklist.pushValue(NewI);
} }
▲ Show 20 Lines • Show All 1,102 Lines • ▼ Show 20 Linesbool VectorCombine::foldShuffleFromReductions(Instruction &I) {
if (NewCost < OldCost) { if (NewCost < OldCost) {
Builder.SetInsertPoint(Shuffle); Builder.SetInsertPoint(Shuffle);
Value *NewShuffle = Builder.CreateShuffleVector( Value *NewShuffle = Builder.CreateShuffleVector(
Shuffle->getOperand(0), Shuffle->getOperand(1), ConcatMask); Shuffle->getOperand(0), Shuffle->getOperand(1), ConcatMask);
LLVM_DEBUG(dbgs() << "Created new shuffle: " << *NewShuffle << "\n"); LLVM_DEBUG(dbgs() << "Created new shuffle: " << *NewShuffle << "\n");
replaceValue(*Shuffle, *NewShuffle); replaceValue(*Shuffle, *NewShuffle);
} }
return false; // See if we can re-use foldSelectShuffle, getting it to reduce the size of
// the shuffle into a nicer order, as it can ignore the order of the shuffles.
return foldSelectShuffle(*Shuffle, true);
} }
/// This method looks for groups of shuffles acting on binops, of the form: /// This method looks for groups of shuffles acting on binops, of the form:
/// %x = shuffle ... /// %x = shuffle ...
/// %y = shuffle ... /// %y = shuffle ...
/// %a = binop %x, %y /// %a = binop %x, %y
/// %b = binop %x, %y /// %b = binop %x, %y
/// shuffle %a, %b, selectmask /// shuffle %a, %b, selectmask
/// We may, especially if the shuffle is wider than legal, be able to convert /// We may, especially if the shuffle is wider than legal, be able to convert
/// the shuffle to a form where only parts of a and b need to be computed. On /// the shuffle to a form where only parts of a and b need to be computed. On
/// architectures with no obvious "select" shuffle, this can reduce the total /// architectures with no obvious "select" shuffle, this can reduce the total
/// number of operations if the target reports them as cheaper. /// number of operations if the target reports them as cheaper.
bool VectorCombine::foldSelectShuffle(Instruction &I) { bool VectorCombine::foldSelectShuffle(Instruction &I, bool FromReduction) {
auto *SVI = dyn_cast<ShuffleVectorInst>(&I); auto *SVI = dyn_cast<ShuffleVectorInst>(&I);
auto *VT = dyn_cast<FixedVectorType>(I.getType()); auto *VT = dyn_cast<FixedVectorType>(I.getType());
if (!SVI || !VT) if (!SVI || !VT)
return false; return false;
auto *Op0 = dyn_cast<Instruction>(SVI->getOperand(0)); auto *Op0 = dyn_cast<Instruction>(SVI->getOperand(0));
auto *Op1 = dyn_cast<Instruction>(SVI->getOperand(1)); auto *Op1 = dyn_cast<Instruction>(SVI->getOperand(1));
if (!Op0 || !Op1 || Op0 == Op1 || !Op0->isBinaryOp() || !Op1->isBinaryOp() || if (!Op0 || !Op1 || Op0 == Op1 || !Op0->isBinaryOp() || !Op1->isBinaryOp() ||
VT != Op0->getType()) VT != Op0->getType())
Show All 22 Lines for (auto *U : I->users()) {
return false; return false;
if (find(Shuffles, SV) == Shuffles.end()) if (find(Shuffles, SV) == Shuffles.end())
Shuffles.push_back(SV); Shuffles.push_back(SV);
} }
return true; return true;
}; };
if (!collectShuffles(Op0) || !collectShuffles(Op1)) if (!collectShuffles(Op0) || !collectShuffles(Op1))
return false; return false;
// From a reduction, we need to be processing a single shuffle, otherwise the
// other uses will not be lane-invariant.
if (FromReduction && Shuffles.size() > 1)
return false;
// For each of the output shuffles, we try to sort all the first vector // For each of the output shuffles, we try to sort all the first vector
// elements to the beginning, followed by the second array elements at the // elements to the beginning, followed by the second array elements at the
// end. If the binops are legalized to smaller vectors, this may reduce total // end. If the binops are legalized to smaller vectors, this may reduce total
// number of binops. We compute the ReconstructMask mask needed to convert // number of binops. We compute the ReconstructMask mask needed to convert
// back to the original lane order. // back to the original lane order.
SmallVector<int> V1, V2; SmallVector<int> V1, V2;
SmallVector<SmallVector<int>> ReconstructMasks; SmallVector<SmallVector<int>> ReconstructMasks;
Show All 37 Lines for (unsigned I = 0; I < Mask.size(); I++) {
ReconstructMask.push_back(NumElts + It - V2.begin()); ReconstructMask.push_back(NumElts + It - V2.begin());
else { else {
ReconstructMask.push_back(NumElts + V2.size()); ReconstructMask.push_back(NumElts + V2.size());
V2.push_back(Mask[I] - NumElts); V2.push_back(Mask[I] - NumElts);
} }
} }
} }
// For reductions, we know that the lane ordering out doesn't alter the
// result. In-order can help simplify the shuffle away.
if (FromReduction)
sort(ReconstructMask);
ReconstructMasks.push_back(ReconstructMask); ReconstructMasks.push_back(ReconstructMask);
} }
// If the Maximum element used from V1 and V2 are not larger than the new // If the Maximum element used from V1 and V2 are not larger than the new
// vectors, the vectors are already packes and performing the optimization // vectors, the vectors are already packes and performing the optimization
// again will likely not help any further. This also prevents us from getting // again will likely not help any further. This also prevents us from getting
// stuck in a cycle in case the costs do not also rule it out. // stuck in a cycle in case the costs do not also rule it out.
if (V1.empty() || V2.empty() || if (V1.empty() || V2.empty() ||
▲ Show 20 Lines • Show All 218 Lines • Show Last 20 Lines

llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll

Show All 16 Lines;
%a = add nsw <16 x i32> %s1, %s2 %a = add nsw <16 x i32> %s1, %s2
%b = sub nsw <16 x i32> %s1, %s2 %b = sub nsw <16 x i32> %s1, %s2
%s3 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 21, i32 22, i32 7, i32 24, i32 25, i32 10, i32 27, i32 28, i32 13, i32 30, i32 31> %s3 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 21, i32 22, i32 7, i32 24, i32 25, i32 10, i32 27, i32 28, i32 13, i32 30, i32 31>
ret <16 x i32> %s3 ret <16 x i32> %s3
} }
define i32 @test1_reduce(<16 x i32> %x, <16 x i32> %y) { define i32 @test1_reduce(<16 x i32> %x, <16 x i32> %y) {
; CHECK-LABEL: @test1_reduce( ; CHECK-LABEL: @test1_reduce(
; CHECK-NEXT: [[S1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 20, i32 21, i32 6, i32 23, i32 24, i32 9, i32 26, i32 27, i32 12, i32 29, i32 30, i32 15> ; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 20, i32 23, i32 26, i32 29, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[S2:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 31, i32 28, i32 25, i32 22, i32 5, i32 4, i32 19, i32 8, i32 7, i32 18, i32 11, i32 10, i32 17, i32 14, i32 13, i32 16> ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 21, i32 6, i32 24, i32 9, i32 27, i32 12, i32 30, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[A:%.*]] = add nsw <16 x i32> [[S1]], [[S2]] ; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 31, i32 28, i32 25, i32 22, i32 5, i32 8, i32 11, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[B:%.*]] = sub nsw <16 x i32> [[S1]], [[S2]] ; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 4, i32 19, i32 7, i32 18, i32 10, i32 17, i32 13, i32 16, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[S3:%.*]] = shufflevector <16 x i32> [[A]], <16 x i32> [[B]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 21, i32 22, i32 7, i32 24, i32 25, i32 10, i32 27, i32 28, i32 13, i32 30, i32 31> ; CHECK-NEXT: [[TMP5:%.*]] = add nsw <16 x i32> [[TMP1]], [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = sub nsw <16 x i32> [[TMP2]], [[TMP4]]
; CHECK-NEXT: [[S3:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[S3]]) ; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[S3]])
; CHECK-NEXT: ret i32 [[R]] ; CHECK-NEXT: ret i32 [[R]]
; ;
%s1 = shufflevector <16 x i32> %x, <16 x i32> %y, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 20, i32 21, i32 6, i32 23, i32 24, i32 9, i32 26, i32 27, i32 12, i32 29, i32 30, i32 15> %s1 = shufflevector <16 x i32> %x, <16 x i32> %y, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 20, i32 21, i32 6, i32 23, i32 24, i32 9, i32 26, i32 27, i32 12, i32 29, i32 30, i32 15>
%s2 = shufflevector <16 x i32> %y, <16 x i32> %x, <16 x i32> <i32 31, i32 28, i32 25, i32 22, i32 5, i32 4, i32 19, i32 8, i32 7, i32 18, i32 11, i32 10, i32 17, i32 14, i32 13, i32 16> %s2 = shufflevector <16 x i32> %y, <16 x i32> %x, <16 x i32> <i32 31, i32 28, i32 25, i32 22, i32 5, i32 4, i32 19, i32 8, i32 7, i32 18, i32 11, i32 10, i32 17, i32 14, i32 13, i32 16>
%a = add nsw <16 x i32> %s1, %s2 %a = add nsw <16 x i32> %s1, %s2
%b = sub nsw <16 x i32> %s1, %s2 %b = sub nsw <16 x i32> %s1, %s2
%s3 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 21, i32 22, i32 7, i32 24, i32 25, i32 10, i32 27, i32 28, i32 13, i32 30, i32 31> %s3 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 21, i32 22, i32 7, i32 24, i32 25, i32 10, i32 27, i32 28, i32 13, i32 30, i32 31>
▲ Show 20 Lines • Show All 469 Lines • ▼ Show 20 Lines
; CHECK-NEXT: [[TMP60:%.*]] = add nsw <16 x i32> [[TMP59]], [[REORDER]] ; CHECK-NEXT: [[TMP60:%.*]] = add nsw <16 x i32> [[TMP59]], [[REORDER]]
; CHECK-NEXT: [[TMP61:%.*]] = sub nsw <16 x i32> [[TMP59]], [[REORDER]] ; CHECK-NEXT: [[TMP61:%.*]] = sub nsw <16 x i32> [[TMP59]], [[REORDER]]
; CHECK-NEXT: [[TMP62:%.*]] = shufflevector <16 x i32> [[TMP60]], <16 x i32> [[TMP61]], <16 x i32> <i32 7, i32 3, i32 15, i32 11, i32 18, i32 22, i32 30, i32 26, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> ; CHECK-NEXT: [[TMP62:%.*]] = shufflevector <16 x i32> [[TMP60]], <16 x i32> [[TMP61]], <16 x i32> <i32 7, i32 3, i32 15, i32 11, i32 18, i32 22, i32 30, i32 26, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP63:%.*]] = shufflevector <16 x i32> [[TMP60]], <16 x i32> [[TMP61]], <16 x i32> <i32 1, i32 5, i32 13, i32 9, i32 16, i32 20, i32 28, i32 24, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> ; CHECK-NEXT: [[TMP63:%.*]] = shufflevector <16 x i32> [[TMP60]], <16 x i32> [[TMP61]], <16 x i32> <i32 1, i32 5, i32 13, i32 9, i32 16, i32 20, i32 28, i32 24, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP64:%.*]] = shufflevector <16 x i32> [[TMP60]], <16 x i32> [[TMP61]], <16 x i32> <i32 5, i32 1, i32 13, i32 9, i32 16, i32 20, i32 28, i32 24, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> ; CHECK-NEXT: [[TMP64:%.*]] = shufflevector <16 x i32> [[TMP60]], <16 x i32> [[TMP61]], <16 x i32> <i32 5, i32 1, i32 13, i32 9, i32 16, i32 20, i32 28, i32 24, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP65:%.*]] = shufflevector <16 x i32> [[TMP60]], <16 x i32> [[TMP61]], <16 x i32> <i32 3, i32 7, i32 15, i32 11, i32 18, i32 22, i32 30, i32 26, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> ; CHECK-NEXT: [[TMP65:%.*]] = shufflevector <16 x i32> [[TMP60]], <16 x i32> [[TMP61]], <16 x i32> <i32 3, i32 7, i32 15, i32 11, i32 18, i32 22, i32 30, i32 26, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP66:%.*]] = add nsw <16 x i32> [[TMP62]], [[TMP64]] ; CHECK-NEXT: [[TMP66:%.*]] = add nsw <16 x i32> [[TMP62]], [[TMP64]]
; CHECK-NEXT: [[TMP67:%.*]] = sub nsw <16 x i32> [[TMP63]], [[TMP65]] ; CHECK-NEXT: [[TMP67:%.*]] = sub nsw <16 x i32> [[TMP63]], [[TMP65]]
; CHECK-NEXT: [[TMP68:%.*]] = shufflevector <16 x i32> [[TMP66]], <16 x i32> [[TMP67]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22> ; CHECK-NEXT: [[TMP68:%.*]] = shufflevector <16 x i32> [[TMP66]], <16 x i32> [[TMP67]], <16 x i32> <i32 1, i32 4, i32 16, i32 20, i32 3, i32 7, i32 19, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[REORDER2:%.*]] = shufflevector <16 x i32> [[TMP66]], <16 x i32> [[TMP67]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23> ; CHECK-NEXT: [[TMP69:%.*]] = shufflevector <16 x i32> [[TMP66]], <16 x i32> [[TMP67]], <16 x i32> <i32 0, i32 5, i32 17, i32 21, i32 2, i32 6, i32 18, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP69:%.*]] = add nsw <16 x i32> [[TMP68]], [[REORDER2]] ; CHECK-NEXT: [[TMP70:%.*]] = shufflevector <16 x i32> [[TMP66]], <16 x i32> [[TMP67]], <16 x i32> <i32 0, i32 5, i32 17, i32 21, i32 2, i32 6, i32 18, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP70:%.*]] = sub nsw <16 x i32> [[TMP68]], [[REORDER2]] ; CHECK-NEXT: [[TMP71:%.*]] = shufflevector <16 x i32> [[TMP66]], <16 x i32> [[TMP67]], <16 x i32> <i32 1, i32 4, i32 16, i32 20, i32 3, i32 7, i32 19, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP71:%.*]] = shufflevector <16 x i32> [[TMP69]], <16 x i32> [[TMP70]], <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 20, i32 5, i32 6, i32 23, i32 24, i32 9, i32 10, i32 27, i32 28, i32 13, i32 14, i32 31> ; CHECK-NEXT: [[TMP72:%.*]] = add nsw <16 x i32> [[TMP68]], [[TMP70]]
; CHECK-NEXT: [[REORDER3:%.*]] = shufflevector <16 x i32> [[TMP69]], <16 x i32> [[TMP70]], <16 x i32> <i32 2, i32 19, i32 0, i32 17, i32 23, i32 6, i32 5, i32 20, i32 27, i32 10, i32 9, i32 24, i32 31, i32 14, i32 13, i32 28> ; CHECK-NEXT: [[TMP73:%.*]] = sub nsw <16 x i32> [[TMP69]], [[TMP71]]
; CHECK-NEXT: [[TMP72:%.*]] = add nsw <16 x i32> [[TMP71]], [[REORDER3]] ; CHECK-NEXT: [[TMP74:%.*]] = shufflevector <16 x i32> [[TMP72]], <16 x i32> [[TMP73]], <16 x i32> <i32 0, i32 16, i32 17, i32 1, i32 18, i32 2, i32 19, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP73:%.*]] = sub nsw <16 x i32> [[TMP71]], [[REORDER3]] ; CHECK-NEXT: [[TMP75:%.*]] = shufflevector <16 x i32> [[TMP72]], <16 x i32> [[TMP73]], <16 x i32> <i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP74:%.*]] = shufflevector <16 x i32> [[TMP72]], <16 x i32> [[TMP73]], <16 x i32> <i32 0, i32 1, i32 18, i32 19, i32 4, i32 5, i32 22, i32 23, i32 8, i32 9, i32 26, i32 27, i32 12, i32 13, i32 30, i32 31> ; CHECK-NEXT: [[TMP76:%.*]] = shufflevector <16 x i32> [[TMP72]], <16 x i32> [[TMP73]], <16 x i32> <i32 4, i32 20, i32 21, i32 5, i32 22, i32 6, i32 23, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP75:%.*]] = lshr <16 x i32> [[TMP74]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15> ; CHECK-NEXT: [[TMP77:%.*]] = shufflevector <16 x i32> [[TMP72]], <16 x i32> [[TMP73]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP76:%.*]] = and <16 x i32> [[TMP75]], <i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537> ; CHECK-NEXT: [[TMP78:%.*]] = add nsw <16 x i32> [[TMP74]], [[TMP76]]
; CHECK-NEXT: [[TMP77:%.*]] = mul nuw <16 x i32> [[TMP76]], <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535> ; CHECK-NEXT: [[TMP79:%.*]] = sub nsw <16 x i32> [[TMP75]], [[TMP77]]
; CHECK-NEXT: [[TMP78:%.*]] = add <16 x i32> [[TMP77]], [[TMP74]] ; CHECK-NEXT: [[TMP80:%.*]] = shufflevector <16 x i32> [[TMP78]], <16 x i32> [[TMP79]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
; CHECK-NEXT: [[TMP79:%.*]] = xor <16 x i32> [[TMP78]], [[TMP77]] ; CHECK-NEXT: [[TMP81:%.*]] = lshr <16 x i32> [[TMP80]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
; CHECK-NEXT: [[TMP80:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP79]]) ; CHECK-NEXT: [[TMP82:%.*]] = and <16 x i32> [[TMP81]], <i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537>
; CHECK-NEXT: [[CONV118:%.*]] = and i32 [[TMP80]], 65535 ; CHECK-NEXT: [[TMP83:%.*]] = mul nuw <16 x i32> [[TMP82]], <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[TMP80]], 16 ; CHECK-NEXT: [[TMP84:%.*]] = add <16 x i32> [[TMP83]], [[TMP80]]
; CHECK-NEXT: [[TMP85:%.*]] = xor <16 x i32> [[TMP84]], [[TMP83]]
; CHECK-NEXT: [[TMP86:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP85]])
; CHECK-NEXT: [[CONV118:%.*]] = and i32 [[TMP86]], 65535
; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[TMP86]], 16
; CHECK-NEXT: [[ADD119:%.*]] = add nuw nsw i32 [[CONV118]], [[SHR]] ; CHECK-NEXT: [[ADD119:%.*]] = add nuw nsw i32 [[CONV118]], [[SHR]]
; CHECK-NEXT: [[SHR120:%.*]] = lshr i32 [[ADD119]], 1 ; CHECK-NEXT: [[SHR120:%.*]] = lshr i32 [[ADD119]], 1
; CHECK-NEXT: ret i32 [[SHR120]] ; CHECK-NEXT: ret i32 [[SHR120]]
; ;
entry: entry:
%idx.ext = sext i32 %st1 to i64 %idx.ext = sext i32 %st1 to i64
%idx.ext63 = sext i32 %st2 to i64 %idx.ext63 = sext i32 %st2 to i64
%arrayidx3 = getelementptr inbounds i8, i8* %p1, i64 4 %arrayidx3 = getelementptr inbounds i8, i8* %p1, i64 4
▲ Show 20 Lines • Show All 174 Lines • ▼ Show 20 Lines
; CHECK-NEXT: [[TMP52:%.*]] = add nsw <16 x i32> [[TMP51]], [[REORDER]] ; CHECK-NEXT: [[TMP52:%.*]] = add nsw <16 x i32> [[TMP51]], [[REORDER]]
; CHECK-NEXT: [[TMP53:%.*]] = sub nsw <16 x i32> [[TMP51]], [[REORDER]] ; CHECK-NEXT: [[TMP53:%.*]] = sub nsw <16 x i32> [[TMP51]], [[REORDER]]
; CHECK-NEXT: [[TMP54:%.*]] = shufflevector <16 x i32> [[TMP52]], <16 x i32> [[TMP53]], <16 x i32> <i32 7, i32 3, i32 15, i32 11, i32 18, i32 22, i32 30, i32 26, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> ; CHECK-NEXT: [[TMP54:%.*]] = shufflevector <16 x i32> [[TMP52]], <16 x i32> [[TMP53]], <16 x i32> <i32 7, i32 3, i32 15, i32 11, i32 18, i32 22, i32 30, i32 26, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP55:%.*]] = shufflevector <16 x i32> [[TMP52]], <16 x i32> [[TMP53]], <16 x i32> <i32 1, i32 5, i32 13, i32 9, i32 16, i32 20, i32 28, i32 24, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> ; CHECK-NEXT: [[TMP55:%.*]] = shufflevector <16 x i32> [[TMP52]], <16 x i32> [[TMP53]], <16 x i32> <i32 1, i32 5, i32 13, i32 9, i32 16, i32 20, i32 28, i32 24, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP56:%.*]] = shufflevector <16 x i32> [[TMP52]], <16 x i32> [[TMP53]], <16 x i32> <i32 5, i32 1, i32 13, i32 9, i32 16, i32 20, i32 28, i32 24, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> ; CHECK-NEXT: [[TMP56:%.*]] = shufflevector <16 x i32> [[TMP52]], <16 x i32> [[TMP53]], <16 x i32> <i32 5, i32 1, i32 13, i32 9, i32 16, i32 20, i32 28, i32 24, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP57:%.*]] = shufflevector <16 x i32> [[TMP52]], <16 x i32> [[TMP53]], <16 x i32> <i32 3, i32 7, i32 15, i32 11, i32 18, i32 22, i32 30, i32 26, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> ; CHECK-NEXT: [[TMP57:%.*]] = shufflevector <16 x i32> [[TMP52]], <16 x i32> [[TMP53]], <16 x i32> <i32 3, i32 7, i32 15, i32 11, i32 18, i32 22, i32 30, i32 26, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP58:%.*]] = add nsw <16 x i32> [[TMP54]], [[TMP56]] ; CHECK-NEXT: [[TMP58:%.*]] = add nsw <16 x i32> [[TMP54]], [[TMP56]]
; CHECK-NEXT: [[TMP59:%.*]] = sub nsw <16 x i32> [[TMP55]], [[TMP57]] ; CHECK-NEXT: [[TMP59:%.*]] = sub nsw <16 x i32> [[TMP55]], [[TMP57]]
; CHECK-NEXT: [[TMP60:%.*]] = shufflevector <16 x i32> [[TMP58]], <16 x i32> [[TMP59]], <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22> ; CHECK-NEXT: [[TMP60:%.*]] = shufflevector <16 x i32> [[TMP58]], <16 x i32> [[TMP59]], <16 x i32> <i32 1, i32 4, i32 16, i32 20, i32 3, i32 7, i32 19, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[REORDER192:%.*]] = shufflevector <16 x i32> [[TMP58]], <16 x i32> [[TMP59]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23> ; CHECK-NEXT: [[TMP61:%.*]] = shufflevector <16 x i32> [[TMP58]], <16 x i32> [[TMP59]], <16 x i32> <i32 0, i32 5, i32 17, i32 21, i32 2, i32 6, i32 18, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP61:%.*]] = add nsw <16 x i32> [[TMP60]], [[REORDER192]] ; CHECK-NEXT: [[TMP62:%.*]] = shufflevector <16 x i32> [[TMP58]], <16 x i32> [[TMP59]], <16 x i32> <i32 0, i32 5, i32 17, i32 21, i32 2, i32 6, i32 18, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP62:%.*]] = sub nsw <16 x i32> [[TMP60]], [[REORDER192]] ; CHECK-NEXT: [[TMP63:%.*]] = shufflevector <16 x i32> [[TMP58]], <16 x i32> [[TMP59]], <16 x i32> <i32 1, i32 4, i32 16, i32 20, i32 3, i32 7, i32 19, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP63:%.*]] = shufflevector <16 x i32> [[TMP61]], <16 x i32> [[TMP62]], <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 20, i32 5, i32 6, i32 23, i32 24, i32 9, i32 10, i32 27, i32 28, i32 13, i32 14, i32 31> ; CHECK-NEXT: [[TMP64:%.*]] = add nsw <16 x i32> [[TMP60]], [[TMP62]]
; CHECK-NEXT: [[REORDER193:%.*]] = shufflevector <16 x i32> [[TMP61]], <16 x i32> [[TMP62]], <16 x i32> <i32 2, i32 19, i32 0, i32 17, i32 23, i32 6, i32 5, i32 20, i32 27, i32 10, i32 9, i32 24, i32 31, i32 14, i32 13, i32 28> ; CHECK-NEXT: [[TMP65:%.*]] = sub nsw <16 x i32> [[TMP61]], [[TMP63]]
; CHECK-NEXT: [[TMP64:%.*]] = add nsw <16 x i32> [[TMP63]], [[REORDER193]] ; CHECK-NEXT: [[TMP66:%.*]] = shufflevector <16 x i32> [[TMP64]], <16 x i32> [[TMP65]], <16 x i32> <i32 0, i32 16, i32 17, i32 1, i32 18, i32 2, i32 19, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP65:%.*]] = sub nsw <16 x i32> [[TMP63]], [[REORDER193]] ; CHECK-NEXT: [[TMP67:%.*]] = shufflevector <16 x i32> [[TMP64]], <16 x i32> [[TMP65]], <16 x i32> <i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP66:%.*]] = shufflevector <16 x i32> [[TMP64]], <16 x i32> [[TMP65]], <16 x i32> <i32 0, i32 1, i32 18, i32 19, i32 4, i32 5, i32 22, i32 23, i32 8, i32 9, i32 26, i32 27, i32 12, i32 13, i32 30, i32 31> ; CHECK-NEXT: [[TMP68:%.*]] = shufflevector <16 x i32> [[TMP64]], <16 x i32> [[TMP65]], <16 x i32> <i32 4, i32 20, i32 21, i32 5, i32 22, i32 6, i32 23, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP67:%.*]] = lshr <16 x i32> [[TMP66]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15> ; CHECK-NEXT: [[TMP69:%.*]] = shufflevector <16 x i32> [[TMP64]], <16 x i32> [[TMP65]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP68:%.*]] = and <16 x i32> [[TMP67]], <i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537> ; CHECK-NEXT: [[TMP70:%.*]] = add nsw <16 x i32> [[TMP66]], [[TMP68]]
; CHECK-NEXT: [[TMP69:%.*]] = mul nuw <16 x i32> [[TMP68]], <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535> ; CHECK-NEXT: [[TMP71:%.*]] = sub nsw <16 x i32> [[TMP67]], [[TMP69]]
; CHECK-NEXT: [[TMP70:%.*]] = add <16 x i32> [[TMP69]], [[TMP66]] ; CHECK-NEXT: [[TMP72:%.*]] = shufflevector <16 x i32> [[TMP70]], <16 x i32> [[TMP71]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
; CHECK-NEXT: [[TMP71:%.*]] = xor <16 x i32> [[TMP70]], [[TMP69]] ; CHECK-NEXT: [[TMP73:%.*]] = lshr <16 x i32> [[TMP72]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
; CHECK-NEXT: [[TMP72:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP71]]) ; CHECK-NEXT: [[TMP74:%.*]] = and <16 x i32> [[TMP73]], <i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537, i32 65537>
; CHECK-NEXT: [[CONV118:%.*]] = and i32 [[TMP72]], 65535 ; CHECK-NEXT: [[TMP75:%.*]] = mul nuw <16 x i32> [[TMP74]], <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[TMP72]], 16 ; CHECK-NEXT: [[TMP76:%.*]] = add <16 x i32> [[TMP75]], [[TMP72]]
; CHECK-NEXT: [[TMP77:%.*]] = xor <16 x i32> [[TMP76]], [[TMP75]]
; CHECK-NEXT: [[TMP78:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP77]])
; CHECK-NEXT: [[CONV118:%.*]] = and i32 [[TMP78]], 65535
; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[TMP78]], 16
; CHECK-NEXT: [[ADD119:%.*]] = add nuw nsw i32 [[CONV118]], [[SHR]] ; CHECK-NEXT: [[ADD119:%.*]] = add nuw nsw i32 [[CONV118]], [[SHR]]
; CHECK-NEXT: [[SHR120:%.*]] = lshr i32 [[ADD119]], 1 ; CHECK-NEXT: [[SHR120:%.*]] = lshr i32 [[ADD119]], 1
; CHECK-NEXT: ret i32 [[SHR120]] ; CHECK-NEXT: ret i32 [[SHR120]]
; ;
entry: entry:
%idx.ext = sext i32 %i_pix1 to i64 %idx.ext = sext i32 %i_pix1 to i64
%idx.ext63 = sext i32 %i_pix2 to i64 %idx.ext63 = sext i32 %i_pix2 to i64
%arrayidx3 = getelementptr inbounds i8, ptr %pix1, i64 4 %arrayidx3 = getelementptr inbounds i8, ptr %pix1, i64 4
▲ Show 20 Lines • Show All 95 Lines • Show Last 20 Lines