This is an archive of the discontinued LLVM Phabricator instance.

Differential D95107

[AArch64][GlobalISel] Allow vector store legalization into 128-bit-wide types
ClosedPublic

Authored by paquette on Jan 20 2021, 5:40 PM.

Details

Reviewers
aemerson
Commits
rG3d25fdc5c21f: [AArch64][GlobalISel] Allow vector store legalization into 128-bit-wide types
Summary

We are allowed to store 128-bit-wide values using the q registers on AArch64.

GlobalISel was clamping the number of elements in vector stores into 64 bits instead.

This results in some poor codegen in cases like below:

https://godbolt.org/z/E56dq8

; SDAG uses a stp + q registers in both cases here.
define void @float(<16 x float> %val, <16 x float>* %ptr) {
    store <16 x float> %val, <16 x float>* %ptr
    ret void
}

define void @double(<8 x double> %val, <8 x double>* %ptr) {
    store <8 x double> %val, <8 x double>* %ptr
    ret void
}

This adds similar legalization for vector stores with s8 and s16 elements.

Diff Detail

Event Timeline

paquette created this revision.Jan 20 2021, 5:40 PM
Herald added subscribers: danielkiss, hiraditya, kristof.beyls, rovka. · View Herald TranscriptJan 20 2021, 5:40 PM
paquette requested review of this revision.Jan 20 2021, 5:40 PM
Herald added a project: Restricted Project. · View Herald TranscriptJan 20 2021, 5:40 PM
aemerson accepted this revision.Jan 22 2021, 10:59 AM

Oh dear, how did we miss that. LGTM.

This revision is now accepted and ready to land.Jan 22 2021, 10:59 AM
Closed by commit rG3d25fdc5c21f: [AArch64][GlobalISel] Allow vector store legalization into 128-bit-wide types (authored by paquette). · Explain WhyJan 27 2021, 2:53 PM
This revision was automatically updated to reflect the committed changes.
paquette added a commit: rG3d25fdc5c21f: [AArch64][GlobalISel] Allow vector store legalization into 128-bit-wide types.

Revision Contents

PathSize
llvm/
lib/
Target/
AArch64/
GISel/
7 lines
test/
CodeGen/
AArch64/
GlobalISel/
14 lines
14 lines
93 lines

Diff 319685

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

Show First 20 Lines • Show All 313 Lines • ▼ Show 20 Lines getActionDefinitionsBuilder(G_STORE)
{v4s32, p0, 128, 8}, {v4s32, p0, 128, 8},
{v2s64, p0, 128, 8}}) {v2s64, p0, 128, 8}})
.clampScalar(0, s8, s64) .clampScalar(0, s8, s64)
.lowerIfMemSizeNotPow2() .lowerIfMemSizeNotPow2()
.lowerIf([=](const LegalityQuery &Query) { .lowerIf([=](const LegalityQuery &Query) {
return Query.Types[0].isScalar() && return Query.Types[0].isScalar() &&
Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits; Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits;
}) })
.clampMaxNumElements(0, s32, 2) // Maximum: sN * k = 128
.clampMaxNumElements(0, s64, 1) .clampMaxNumElements(0, s8, 16)
.clampMaxNumElements(0, s16, 8)
.clampMaxNumElements(0, s32, 4)
.clampMaxNumElements(0, s64, 2)
.customIf(IsPtrVecPred); .customIf(IsPtrVecPred);
// Constants // Constants
getActionDefinitionsBuilder(G_CONSTANT) getActionDefinitionsBuilder(G_CONSTANT)
.legalFor({p0, s8, s16, s32, s64}) .legalFor({p0, s8, s16, s32, s64})
.clampScalar(0, s8, s64) .clampScalar(0, s8, s64)
.widenScalarToNextPow2(0); .widenScalarToNextPow2(0);
getActionDefinitionsBuilder(G_FCONSTANT) getActionDefinitionsBuilder(G_FCONSTANT)
▲ Show 20 Lines • Show All 602 Lines • Show Last 20 Lines

llvm/test/CodeGen/AArch64/GlobalISel/legalize-fpext.mir

Show All 13 Lines bb.1:
; CHECK-LABEL: name: fpext_v4s64_v4s32 ; CHECK-LABEL: name: fpext_v4s64_v4s32
; CHECK: liveins: $q0, $x0 ; CHECK: liveins: $q0, $x0
; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x0 ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x0
; CHECK: [[UV:%[0-9]+]]:_(<2 x s32>), [[UV1:%[0-9]+]]:_(<2 x s32>) = G_UNMERGE_VALUES [[COPY]](<4 x s32>) ; CHECK: [[UV:%[0-9]+]]:_(<2 x s32>), [[UV1:%[0-9]+]]:_(<2 x s32>) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
; CHECK: [[FPEXT:%[0-9]+]]:_(<2 x s64>) = G_FPEXT [[UV]](<2 x s32>) ; CHECK: [[FPEXT:%[0-9]+]]:_(<2 x s64>) = G_FPEXT [[UV]](<2 x s32>)
; CHECK: [[FPEXT1:%[0-9]+]]:_(<2 x s64>) = G_FPEXT [[UV1]](<2 x s32>) ; CHECK: [[FPEXT1:%[0-9]+]]:_(<2 x s64>) = G_FPEXT [[UV1]](<2 x s32>)
; CHECK: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[FPEXT]](<2 x s64>) ; CHECK: G_STORE [[FPEXT]](<2 x s64>), [[COPY1]](p0) :: (store 16, align 32)
; CHECK: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[FPEXT1]](<2 x s64>) ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
; CHECK: G_STORE [[UV2]](s64), [[COPY1]](p0) :: (store 8, align 32)
; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s64) ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s64)
; CHECK: G_STORE [[UV3]](s64), [[PTR_ADD]](p0) :: (store 8 + 8) ; CHECK: G_STORE [[FPEXT1]](<2 x s64>), [[PTR_ADD]](p0) :: (store 16 + 16)
; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s64)
; CHECK: G_STORE [[UV4]](s64), [[PTR_ADD1]](p0) :: (store 8 + 16, align 16)
; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
; CHECK: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C2]](s64)
; CHECK: G_STORE [[UV5]](s64), [[PTR_ADD2]](p0) :: (store 8 + 24)
; CHECK: RET_ReallyLR ; CHECK: RET_ReallyLR
%0:_(<4 x s32>) = COPY $q0 %0:_(<4 x s32>) = COPY $q0
%1:_(p0) = COPY $x0 %1:_(p0) = COPY $x0
%2:_(<4 x s64>) = G_FPEXT %0(<4 x s32>) %2:_(<4 x s64>) = G_FPEXT %0(<4 x s32>)
G_STORE %2(<4 x s64>), %1(p0) :: (store 32) G_STORE %2(<4 x s64>), %1(p0) :: (store 32)
RET_ReallyLR RET_ReallyLR
... ...

llvm/test/CodeGen/AArch64/GlobalISel/legalize-fptrunc.mir

Show First 20 Lines • Show All 109 Lines • ▼ Show 20 Lines bb.0:
; CHECK: [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $q2 ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $q2
; CHECK: [[COPY3:%[0-9]+]]:_(<2 x s64>) = COPY $q3 ; CHECK: [[COPY3:%[0-9]+]]:_(<2 x s64>) = COPY $q3
; CHECK: [[COPY4:%[0-9]+]]:_(p0) = COPY $x0 ; CHECK: [[COPY4:%[0-9]+]]:_(p0) = COPY $x0
; CHECK: [[FPTRUNC:%[0-9]+]]:_(<2 x s32>) = G_FPTRUNC [[COPY]](<2 x s64>) ; CHECK: [[FPTRUNC:%[0-9]+]]:_(<2 x s32>) = G_FPTRUNC [[COPY]](<2 x s64>)
; CHECK: [[FPTRUNC1:%[0-9]+]]:_(<2 x s32>) = G_FPTRUNC [[COPY1]](<2 x s64>) ; CHECK: [[FPTRUNC1:%[0-9]+]]:_(<2 x s32>) = G_FPTRUNC [[COPY1]](<2 x s64>)
; CHECK: [[FPTRUNC2:%[0-9]+]]:_(<2 x s32>) = G_FPTRUNC [[COPY2]](<2 x s64>) ; CHECK: [[FPTRUNC2:%[0-9]+]]:_(<2 x s32>) = G_FPTRUNC [[COPY2]](<2 x s64>)
; CHECK: [[FPTRUNC3:%[0-9]+]]:_(<2 x s32>) = G_FPTRUNC [[COPY3]](<2 x s64>) ; CHECK: [[FPTRUNC3:%[0-9]+]]:_(<2 x s32>) = G_FPTRUNC [[COPY3]](<2 x s64>)
; CHECK: [[COPY5:%[0-9]+]]:_(p0) = COPY $x0 ; CHECK: [[COPY5:%[0-9]+]]:_(p0) = COPY $x0
; CHECK: G_STORE [[FPTRUNC]](<2 x s32>), [[COPY5]](p0) :: (store 8, align 32) ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[FPTRUNC]](<2 x s32>), [[FPTRUNC1]](<2 x s32>)
; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 ; CHECK: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[FPTRUNC2]](<2 x s32>), [[FPTRUNC3]](<2 x s32>)
; CHECK: G_STORE [[CONCAT_VECTORS]](<4 x s32>), [[COPY5]](p0) :: (store 16, align 32)
; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY5]], [[C]](s64) ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY5]], [[C]](s64)
; CHECK: G_STORE [[FPTRUNC1]](<2 x s32>), [[PTR_ADD]](p0) :: (store 8 + 8) ; CHECK: G_STORE [[CONCAT_VECTORS1]](<4 x s32>), [[PTR_ADD]](p0) :: (store 16 + 16)
; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY5]], [[C1]](s64)
; CHECK: G_STORE [[FPTRUNC2]](<2 x s32>), [[PTR_ADD1]](p0) :: (store 8 + 16, align 16)
; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
; CHECK: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY5]], [[C2]](s64)
; CHECK: G_STORE [[FPTRUNC3]](<2 x s32>), [[PTR_ADD2]](p0) :: (store 8 + 24)
; CHECK: RET_ReallyLR ; CHECK: RET_ReallyLR
%2:_(<2 x s64>) = COPY $q0 %2:_(<2 x s64>) = COPY $q0
%3:_(<2 x s64>) = COPY $q1 %3:_(<2 x s64>) = COPY $q1
%4:_(<2 x s64>) = COPY $q2 %4:_(<2 x s64>) = COPY $q2
%5:_(<2 x s64>) = COPY $q3 %5:_(<2 x s64>) = COPY $q3
%0:_(<8 x s64>) = G_CONCAT_VECTORS %2(<2 x s64>), %3(<2 x s64>), %4(<2 x s64>), %5(<2 x s64>) %0:_(<8 x s64>) = G_CONCAT_VECTORS %2(<2 x s64>), %3(<2 x s64>), %4(<2 x s64>), %5(<2 x s64>)
%1:_(p0) = COPY $x0 %1:_(p0) = COPY $x0
%6:_(<8 x s32>) = G_FPTRUNC %0(<8 x s64>) %6:_(<8 x s32>) = G_FPTRUNC %0(<8 x s64>)
%7:_(p0) = COPY $x0 %7:_(p0) = COPY $x0
G_STORE %6(<8 x s32>), %7(p0) :: (store 32) G_STORE %6(<8 x s32>), %7(p0) :: (store 32)
RET_ReallyLR RET_ReallyLR
... ...

llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir

Show First 20 Lines • Show All 296 Lines • ▼ Show 20 Lines bb.1:
; CHECK: [[COPY1:%[0-9]+]]:_(<8 x s8>) = COPY $d0 ; CHECK: [[COPY1:%[0-9]+]]:_(<8 x s8>) = COPY $d0
; CHECK: G_STORE [[COPY1]](<8 x s8>), [[COPY]](p0) :: (store 8) ; CHECK: G_STORE [[COPY1]](<8 x s8>), [[COPY]](p0) :: (store 8)
; CHECK: RET_ReallyLR ; CHECK: RET_ReallyLR
%0:_(p0) = COPY $x0 %0:_(p0) = COPY $x0
%1:_(<8 x s8>) = COPY $d0 %1:_(<8 x s8>) = COPY $d0
G_STORE %1(<8 x s8>), %0(p0) :: (store 8) G_STORE %1(<8 x s8>), %0(p0) :: (store 8)
RET_ReallyLR RET_ReallyLR
... ...
---
name: store_32xs8
alignment: 4
tracksRegLiveness: true
machineFunctionInfo: {}
body: |
bb.1:
liveins: $x0
; CHECK-LABEL: name: store_32xs8
; CHECK: liveins: $x0
; CHECK: [[DEF:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
; CHECK: %ptr:_(p0) = COPY $x0
; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
; CHECK: G_STORE [[BUILD_VECTOR]](<16 x s8>), %ptr(p0) :: (store 16, align 32)
; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD %ptr, [[C]](s64)
; CHECK: G_STORE [[BUILD_VECTOR1]](<16 x s8>), [[PTR_ADD]](p0) :: (store 16 + 16)
; CHECK: RET_ReallyLR
%val:_(<32 x s8>) = G_IMPLICIT_DEF
%ptr:_(p0) = COPY $x0
G_STORE %val(<32 x s8>), %ptr(p0) :: (store 32)
RET_ReallyLR
...
---
name: store_16xs16
alignment: 4
tracksRegLiveness: true
machineFunctionInfo: {}
body: |
bb.1:
liveins: $x0
; CHECK-LABEL: name: store_16xs16
; CHECK: liveins: $x0
; CHECK: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; CHECK: %ptr:_(p0) = COPY $x0
; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16), [[DEF]](s16)
; CHECK: G_STORE [[BUILD_VECTOR]](<8 x s16>), %ptr(p0) :: (store 16, align 32)
; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD %ptr, [[C]](s64)
; CHECK: G_STORE [[BUILD_VECTOR1]](<8 x s16>), [[PTR_ADD]](p0) :: (store 16 + 16)
; CHECK: RET_ReallyLR
%val:_(<16 x s16>) = G_IMPLICIT_DEF
%ptr:_(p0) = COPY $x0
G_STORE %val(<16 x s16>), %ptr(p0) :: (store 32)
RET_ReallyLR
...
---
name: store_8xs32
alignment: 4
tracksRegLiveness: true
machineFunctionInfo: {}
body: |
bb.1:
liveins: $x0
; CHECK-LABEL: name: store_8xs32
; CHECK: liveins: $x0
; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
; CHECK: %ptr:_(p0) = COPY $x0
; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
; CHECK: G_STORE [[BUILD_VECTOR]](<4 x s32>), %ptr(p0) :: (store 16, align 32)
; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD %ptr, [[C]](s64)
; CHECK: G_STORE [[BUILD_VECTOR1]](<4 x s32>), [[PTR_ADD]](p0) :: (store 16 + 16)
; CHECK: RET_ReallyLR
%val:_(<8 x s32>) = G_IMPLICIT_DEF
%ptr:_(p0) = COPY $x0
G_STORE %val(<8 x s32>), %ptr(p0) :: (store 32)
RET_ReallyLR
...
---
name: store_4xs64
alignment: 4
tracksRegLiveness: true
machineFunctionInfo: {}
body: |
bb.1:
liveins: $x0
; CHECK-LABEL: name: store_4xs64
; CHECK: liveins: $x0
; CHECK: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
; CHECK: %ptr:_(p0) = COPY $x0
; CHECK: G_STORE [[DEF]](<2 x s64>), %ptr(p0) :: (store 16, align 32)
; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD %ptr, [[C]](s64)
; CHECK: G_STORE [[DEF]](<2 x s64>), [[PTR_ADD]](p0) :: (store 16 + 16)
; CHECK: RET_ReallyLR
%val:_(<4 x s64>) = G_IMPLICIT_DEF
%ptr:_(p0) = COPY $x0
G_STORE %val(<4 x s64>), %ptr(p0) :: (store 32)
RET_ReallyLR