This is an archive of the discontinued LLVM Phabricator instance.

Differential D41786

[SLP] Fix PR35628: Count external uses on extra reduction arguments.
ClosedPublic

Authored by ABataev on Jan 5 2018, 2:09 PM.

Details

Reviewers
mkuper
hfinkel
RKSimon
spatel
Commits
rGc3a8a286c277: Merging r321993: --------------------------------------------------------------…
rG118a0a2c38ff: [SLP] Fix PR35628: Count external uses on extra reduction arguments.
rL322669: Merging r321993:
rL321993: [SLP] Fix PR35628: Count external uses on extra reduction arguments.
Summary

If the vectorized value is marked as extra reduction argument, its users
are not considered as external users. Patch fixes this.

Diff Detail

Repository
rL LLVM

Event Timeline

ABataev created this revision.Jan 5 2018, 2:09 PM
hfinkel accepted this revision.Jan 6 2018, 9:08 PM

LGTM

This revision is now accepted and ready to land.Jan 6 2018, 9:08 PM
Closed by commit rL321993: [SLP] Fix PR35628: Count external uses on extra reduction arguments. (authored by ABataev). · Explain WhyJan 8 2018, 6:34 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents

PathSize
llvm/
trunk/
lib/
Transforms/
Vectorize/
1 line
test/
Transforms/
SLPVectorizer/
X86/
74 lines
64 lines

Diff 128920

llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp

Show First 20 Lines • Show All 1,341 Lines • ▼ Show 20 Lines for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
Value *Scalar = Entry->Scalars[Lane]; Value *Scalar = Entry->Scalars[Lane];
// Check if the scalar is externally used as an extra arg. // Check if the scalar is externally used as an extra arg.
auto ExtI = ExternallyUsedValues.find(Scalar); auto ExtI = ExternallyUsedValues.find(Scalar);
if (ExtI != ExternallyUsedValues.end()) { if (ExtI != ExternallyUsedValues.end()) {
DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane " << DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane " <<
Lane << " from " << *Scalar << ".\n"); Lane << " from " << *Scalar << ".\n");
ExternalUses.emplace_back(Scalar, nullptr, Lane); ExternalUses.emplace_back(Scalar, nullptr, Lane);
continue;
} }
for (User *U : Scalar->users()) { for (User *U : Scalar->users()) {
DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n"); DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n");
Instruction *UserInst = dyn_cast<Instruction>(U); Instruction *UserInst = dyn_cast<Instruction>(U);
if (!UserInst) if (!UserInst)
continue; continue;
▲ Show 20 Lines • Show All 4,670 Lines • Show Last 20 Lines

llvm/trunk/test/Transforms/SLPVectorizer/X86/PR35628_1.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -slp-vectorizer -slp-vectorize-hor -slp-vectorize-hor-store -S < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
define void @mainTest(i32* %ptr) #0 {
; CHECK-LABEL: @mainTest(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32* [[PTR:%.*]], null
; CHECK-NEXT: br i1 [[CMP]], label [[LOOP:%.*]], label [[BAIL_OUT:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[DUMMY_PHI:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[OP_EXTRA5:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[PTR]], i64 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[PTR]], i64 2
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[PTR]], i64 3
; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[PTR]] to <4 x i32>*
; CHECK-NEXT: [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP4]], i32 3
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP4]], i32 2
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP4]], i32 1
; CHECK-NEXT: [[TMP8:%.*]] = mul <4 x i32> [[TMP4]], [[TMP4]]
; CHECK-NEXT: [[TMP9:%.*]] = add i32 1, undef
; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[TMP9]], undef
; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP10]], undef
; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], undef
; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP12]], undef
; CHECK-NEXT: [[TMP14:%.*]] = sext i32 [[TMP6]] to i64
; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP13]], undef
; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP8]], [[RDX_SHUF]]
; CHECK-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[BIN_RDX2:%.*]] = add <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
; CHECK-NEXT: [[OP_EXTRA:%.*]] = add i32 [[TMP16]], 1
; CHECK-NEXT: [[OP_EXTRA3:%.*]] = add i32 [[OP_EXTRA]], [[TMP7]]
; CHECK-NEXT: [[OP_EXTRA4:%.*]] = add i32 [[OP_EXTRA3]], [[TMP6]]
; CHECK-NEXT: [[OP_EXTRA5]] = add i32 [[OP_EXTRA4]], [[TMP5]]
; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP15]], undef
; CHECK-NEXT: br label [[LOOP]]
; CHECK: bail_out:
; CHECK-NEXT: ret void
;
entry:
%cmp = icmp eq i32* %ptr, null
br i1 %cmp, label %loop, label %bail_out
loop:
%dummy_phi = phi i32 [ 1, %entry ], [ %18, %loop ]
%0 = load i32, i32 * %ptr , align 4
%1 = mul i32 %0, %0
%2 = add i32 1, %1
%3 = getelementptr inbounds i32, i32 * %ptr, i64 1
%4 = load i32, i32 * %3 , align 4
%5 = mul i32 %4, %4
%6 = add i32 %2, %4
%7 = add i32 %6, %5
%8 = getelementptr inbounds i32, i32 *%ptr, i64 2
%9 = load i32, i32 * %8 , align 4
%10 = mul i32 %9, %9
%11 = add i32 %7, %9
%12 = add i32 %11, %10
%13 = sext i32 %9 to i64
%14 = getelementptr inbounds i32, i32 *%ptr, i64 3
%15 = load i32, i32 * %14 , align 4
%16 = mul i32 %15, %15
%17 = add i32 %12, %15
%18 = add i32 %17, %16
br label %loop
bail_out:
ret void
}
attributes #0 = { "target-cpu"="westmere" }

llvm/trunk/test/Transforms/SLPVectorizer/X86/PR35628_2.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -slp-vectorizer -slp-vectorize-hor -slp-vectorize-hor-store -S < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=haswell | FileCheck %s
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
define void @test() #0 {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[DUMMY_PHI:%.*]] = phi i64 [ 1, [[ENTRY:%.*]] ], [ [[OP_EXTRA3:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ 2, [[ENTRY]] ], [ [[TMP6:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[DUMMY_ADD:%.*]] = add i16 0, 0
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> undef, i64 [[TMP0]], i32 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[TMP0]], i32 1
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 [[TMP0]], i32 2
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[TMP0]], i32 3
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> <i64 3, i64 2, i64 1, i64 0>, [[TMP4]]
; CHECK-NEXT: [[TMP6]] = extractelement <4 x i64> [[TMP5]], i32 3
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP5]], i32 0
; CHECK-NEXT: [[DUMMY_SHL:%.*]] = shl i64 [[TMP7]], 32
; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> <i64 1, i64 1, i64 1, i64 1>, [[TMP5]]
; CHECK-NEXT: [[TMP9:%.*]] = ashr exact <4 x i64> [[TMP8]], <i64 32, i64 32, i64 32, i64 32>
; CHECK-NEXT: [[SUM1:%.*]] = add i64 undef, undef
; CHECK-NEXT: [[SUM2:%.*]] = add i64 [[SUM1]], undef
; CHECK-NEXT: [[ZSUM:%.*]] = add i64 [[SUM2]], 0
; CHECK-NEXT: [[JOIN:%.*]] = add i64 undef, [[ZSUM]]
; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i64> [[TMP9]], [[RDX_SHUF]]
; CHECK-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <4 x i64> [[BIN_RDX]], <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[BIN_RDX2:%.*]] = add <4 x i64> [[BIN_RDX]], [[RDX_SHUF1]]
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[BIN_RDX2]], i32 0
; CHECK-NEXT: [[OP_EXTRA:%.*]] = add i64 [[TMP10]], 0
; CHECK-NEXT: [[OP_EXTRA3]] = add i64 [[OP_EXTRA]], [[TMP6]]
; CHECK-NEXT: [[LAST:%.*]] = add i64 [[JOIN]], undef
; CHECK-NEXT: br label [[LOOP]]
;
entry:
br label %loop
loop:
%dummy_phi = phi i64 [ 1, %entry ], [ %last, %loop ]
%0 = phi i64 [ 2, %entry ], [ %fork, %loop ]
%inc1 = add i64 %0, 1
%inc2 = add i64 %0, 2
%inc11 = add i64 1, %inc1
%exact1 = ashr exact i64 %inc11, 32
%inc3 = add i64 %0, 3
%dummy_add = add i16 0, 0
%inc12 = add i64 1, %inc2
%exact2 = ashr exact i64 %inc12, 32
%dummy_shl = shl i64 %inc3, 32
%inc13 = add i64 1, %inc3
%exact3 = ashr exact i64 %inc13, 32
%fork = add i64 %0, 0
%sum1 = add i64 %exact3, %exact2
%sum2 = add i64 %sum1, %exact1
%zsum = add i64 %sum2, 0
%sext22 = add i64 1, %fork
%exact4 = ashr exact i64 %sext22, 32
%join = add i64 %fork, %zsum
%last = add i64 %join, %exact4
br label %loop
}