diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -78,6 +78,8 @@ const bool HasFP16 = ST.hasFullFP16(); const LLT &MinFPScalar = HasFP16 ? s16 : s32; + const bool HasCSSC = ST.hasCSSC(); + getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE}) .legalFor({p0, s8, s16, s32, s64}) .legalFor(PackedVectorAllTypeList) @@ -668,7 +670,12 @@ .lowerIf(isVector(0)) .clampScalar(0, s32, s64) .scalarSameSizeAs(1, 0) - .customFor({s32, s64}); + .legalIf([=](const LegalityQuery &Query) { + return (HasCSSC && typeInSet(0, {s32, s64})(Query)); + }) + .customIf([=](const LegalityQuery &Query) { + return (!HasCSSC && typeInSet(0, {s32, s64})(Query)); + }); getActionDefinitionsBuilder(G_SHUFFLE_VECTOR) .legalIf([=](const LegalityQuery &Query) { diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cttz-zero-undef.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cttz-zero-undef.mir --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cttz-zero-undef.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cttz-zero-undef.mir @@ -1,6 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs -run-pass=legalizer %s -o - | FileCheck %s - +# RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs -mattr=+cssc -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=CHECK-CSSC ... --- name: s8 @@ -10,14 +10,25 @@ liveins: $w0 ; CHECK-LABEL: name: s8 ; CHECK: liveins: $w0 - ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF - ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 256 - ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[DEF]], [[C]] - ; CHECK: [[BITREVERSE:%[0-9]+]]:_(s32) = G_BITREVERSE [[OR]] - ; CHECK: [[CTLZ:%[0-9]+]]:_(s32) = G_CTLZ [[BITREVERSE]](s32) - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[CTLZ]](s32) - ; CHECK: $w0 = COPY [[COPY]](s32) - ; CHECK: RET_ReallyLR implicit $w0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 256 + ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[DEF]], [[C]] + ; CHECK-NEXT: [[BITREVERSE:%[0-9]+]]:_(s32) = G_BITREVERSE [[OR]] + ; CHECK-NEXT: [[CTLZ:%[0-9]+]]:_(s32) = G_CTLZ [[BITREVERSE]](s32) + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[CTLZ]](s32) + ; CHECK-NEXT: $w0 = COPY [[COPY]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + ; CHECK-CSSC-LABEL: name: s8 + ; CHECK-CSSC: liveins: $w0 + ; CHECK-CSSC-NEXT: {{ $}} + ; CHECK-CSSC-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF + ; CHECK-CSSC-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 256 + ; CHECK-CSSC-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[DEF]], [[C]] + ; CHECK-CSSC-NEXT: [[CTTZ:%[0-9]+]]:_(s32) = G_CTTZ [[OR]](s32) + ; CHECK-CSSC-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[CTTZ]](s32) + ; CHECK-CSSC-NEXT: $w0 = COPY [[COPY]](s32) + ; CHECK-CSSC-NEXT: RET_ReallyLR implicit $w0 %val:_(s8) = G_IMPLICIT_DEF %cttz:_(s8) = G_CTTZ_ZERO_UNDEF %val(s8) %ext:_(s32) = G_ANYEXT %cttz(s8) @@ -33,14 +44,25 @@ liveins: $w0 ; CHECK-LABEL: name: s16 ; CHECK: liveins: $w0 - ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF - ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65536 - ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[DEF]], [[C]] - ; CHECK: [[BITREVERSE:%[0-9]+]]:_(s32) = G_BITREVERSE [[OR]] - ; CHECK: [[CTLZ:%[0-9]+]]:_(s32) = G_CTLZ [[BITREVERSE]](s32) - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[CTLZ]](s32) - ; CHECK: $w0 = COPY [[COPY]](s32) - ; CHECK: RET_ReallyLR implicit $w0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65536 + ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[DEF]], [[C]] + ; CHECK-NEXT: [[BITREVERSE:%[0-9]+]]:_(s32) = G_BITREVERSE [[OR]] + ; CHECK-NEXT: [[CTLZ:%[0-9]+]]:_(s32) = G_CTLZ [[BITREVERSE]](s32) + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[CTLZ]](s32) + ; CHECK-NEXT: $w0 = COPY [[COPY]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + ; CHECK-CSSC-LABEL: name: s16 + ; CHECK-CSSC: liveins: $w0 + ; CHECK-CSSC-NEXT: {{ $}} + ; CHECK-CSSC-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF + ; CHECK-CSSC-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65536 + ; CHECK-CSSC-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[DEF]], [[C]] + ; CHECK-CSSC-NEXT: [[CTTZ:%[0-9]+]]:_(s32) = G_CTTZ [[OR]](s32) + ; CHECK-CSSC-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[CTTZ]](s32) + ; CHECK-CSSC-NEXT: $w0 = COPY [[COPY]](s32) + ; CHECK-CSSC-NEXT: RET_ReallyLR implicit $w0 %val:_(s16) = G_IMPLICIT_DEF %cttz:_(s16) = G_CTTZ_ZERO_UNDEF %val(s16) %ext:_(s32) = G_ANYEXT %cttz(s16) @@ -58,11 +80,19 @@ ; CHECK-LABEL: name: s32 ; CHECK: liveins: $w0 - ; CHECK: %val:_(s32) = COPY $w0 - ; CHECK: [[BITREVERSE:%[0-9]+]]:_(s32) = G_BITREVERSE %val - ; CHECK: [[CTLZ:%[0-9]+]]:_(s32) = G_CTLZ [[BITREVERSE]](s32) - ; CHECK: $w0 = COPY [[CTLZ]](s32) - ; CHECK: RET_ReallyLR implicit $w0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %val:_(s32) = COPY $w0 + ; CHECK-NEXT: [[BITREVERSE:%[0-9]+]]:_(s32) = G_BITREVERSE %val + ; CHECK-NEXT: [[CTLZ:%[0-9]+]]:_(s32) = G_CTLZ [[BITREVERSE]](s32) + ; CHECK-NEXT: $w0 = COPY [[CTLZ]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + ; CHECK-CSSC-LABEL: name: s32 + ; CHECK-CSSC: liveins: $w0 + ; CHECK-CSSC-NEXT: {{ $}} + ; CHECK-CSSC-NEXT: %val:_(s32) = COPY $w0 + ; CHECK-CSSC-NEXT: [[CTTZ:%[0-9]+]]:_(s32) = G_CTTZ %val(s32) + ; CHECK-CSSC-NEXT: $w0 = COPY [[CTTZ]](s32) + ; CHECK-CSSC-NEXT: RET_ReallyLR implicit $w0 %val:_(s32) = COPY $w0 %1:_(s32) = G_CTTZ_ZERO_UNDEF %val(s32) $w0 = COPY %1(s32) @@ -79,11 +109,19 @@ ; CHECK-LABEL: name: s64 ; CHECK: liveins: $x0 - ; CHECK: %val:_(s64) = COPY $x0 - ; CHECK: [[BITREVERSE:%[0-9]+]]:_(s64) = G_BITREVERSE %val - ; CHECK: [[CTLZ:%[0-9]+]]:_(s64) = G_CTLZ [[BITREVERSE]](s64) - ; CHECK: $x0 = COPY [[CTLZ]](s64) - ; CHECK: RET_ReallyLR implicit $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %val:_(s64) = COPY $x0 + ; CHECK-NEXT: [[BITREVERSE:%[0-9]+]]:_(s64) = G_BITREVERSE %val + ; CHECK-NEXT: [[CTLZ:%[0-9]+]]:_(s64) = G_CTLZ [[BITREVERSE]](s64) + ; CHECK-NEXT: $x0 = COPY [[CTLZ]](s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + ; CHECK-CSSC-LABEL: name: s64 + ; CHECK-CSSC: liveins: $x0 + ; CHECK-CSSC-NEXT: {{ $}} + ; CHECK-CSSC-NEXT: %val:_(s64) = COPY $x0 + ; CHECK-CSSC-NEXT: [[CTTZ:%[0-9]+]]:_(s64) = G_CTTZ %val(s64) + ; CHECK-CSSC-NEXT: $x0 = COPY [[CTTZ]](s64) + ; CHECK-CSSC-NEXT: RET_ReallyLR implicit $x0 %val:_(s64) = COPY $x0 %1:_(s64) = G_CTTZ_ZERO_UNDEF %val(s64) $x0 = COPY %1(s64) diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cttz.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cttz.mir --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cttz.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-cttz.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs -run-pass=legalizer %s -o - | FileCheck %s +# RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs -mattr=+cssc -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=CHECK-CSSC ... --- @@ -19,6 +20,16 @@ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[CTLZ]](s32) ; CHECK-NEXT: $w0 = COPY [[COPY]](s32) ; CHECK-NEXT: RET_ReallyLR implicit $w0 + ; CHECK-CSSC-LABEL: name: s8 + ; CHECK-CSSC: liveins: $w0 + ; CHECK-CSSC-NEXT: {{ $}} + ; CHECK-CSSC-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF + ; CHECK-CSSC-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 256 + ; CHECK-CSSC-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[DEF]], [[C]] + ; CHECK-CSSC-NEXT: [[CTTZ:%[0-9]+]]:_(s32) = G_CTTZ [[OR]](s32) + ; CHECK-CSSC-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[CTTZ]](s32) + ; CHECK-CSSC-NEXT: $w0 = COPY [[COPY]](s32) + ; CHECK-CSSC-NEXT: RET_ReallyLR implicit $w0 %val:_(s8) = G_IMPLICIT_DEF %cttz:_(s8) = G_CTTZ %val(s8) %ext:_(s32) = G_ANYEXT %cttz(s8) @@ -43,6 +54,16 @@ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[CTLZ]](s32) ; CHECK-NEXT: $w0 = COPY [[COPY]](s32) ; CHECK-NEXT: RET_ReallyLR implicit $w0 + ; CHECK-CSSC-LABEL: name: s16 + ; CHECK-CSSC: liveins: $w0 + ; CHECK-CSSC-NEXT: {{ $}} + ; CHECK-CSSC-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF + ; CHECK-CSSC-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65536 + ; CHECK-CSSC-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[DEF]], [[C]] + ; CHECK-CSSC-NEXT: [[CTTZ:%[0-9]+]]:_(s32) = G_CTTZ [[OR]](s32) + ; CHECK-CSSC-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[CTTZ]](s32) + ; CHECK-CSSC-NEXT: $w0 = COPY [[COPY]](s32) + ; CHECK-CSSC-NEXT: RET_ReallyLR implicit $w0 %val:_(s16) = G_IMPLICIT_DEF %cttz:_(s16) = G_CTTZ %val(s16) %ext:_(s32) = G_ANYEXT %cttz(s16) @@ -66,6 +87,13 @@ ; CHECK-NEXT: [[CTLZ:%[0-9]+]]:_(s32) = G_CTLZ [[BITREVERSE]](s32) ; CHECK-NEXT: $w0 = COPY [[CTLZ]](s32) ; CHECK-NEXT: RET_ReallyLR implicit $w0 + ; CHECK-CSSC-LABEL: name: s32 + ; CHECK-CSSC: liveins: $w0 + ; CHECK-CSSC-NEXT: {{ $}} + ; CHECK-CSSC-NEXT: %val:_(s32) = COPY $w0 + ; CHECK-CSSC-NEXT: [[CTTZ:%[0-9]+]]:_(s32) = G_CTTZ %val(s32) + ; CHECK-CSSC-NEXT: $w0 = COPY [[CTTZ]](s32) + ; CHECK-CSSC-NEXT: RET_ReallyLR implicit $w0 %val:_(s32) = COPY $w0 %1:_(s32) = G_CTTZ %val(s32) $w0 = COPY %1(s32) @@ -88,6 +116,13 @@ ; CHECK-NEXT: [[CTLZ:%[0-9]+]]:_(s64) = G_CTLZ [[BITREVERSE]](s64) ; CHECK-NEXT: $x0 = COPY [[CTLZ]](s64) ; CHECK-NEXT: RET_ReallyLR implicit $x0 + ; CHECK-CSSC-LABEL: name: s64 + ; CHECK-CSSC: liveins: $x0 + ; CHECK-CSSC-NEXT: {{ $}} + ; CHECK-CSSC-NEXT: %val:_(s64) = COPY $x0 + ; CHECK-CSSC-NEXT: [[CTTZ:%[0-9]+]]:_(s64) = G_CTTZ %val(s64) + ; CHECK-CSSC-NEXT: $x0 = COPY [[CTTZ]](s64) + ; CHECK-CSSC-NEXT: RET_ReallyLR implicit $x0 %val:_(s64) = COPY $x0 %1:_(s64) = G_CTTZ %val(s64) $x0 = COPY %1(s64) @@ -113,8 +148,21 @@ ; CHECK-NEXT: [[CTPOP:%[0-9]+]]:_(<4 x s32>) = G_CTPOP [[AND]](<4 x s32>) ; CHECK-NEXT: $q0 = COPY [[CTPOP]](<4 x s32>) ; CHECK-NEXT: RET_ReallyLR implicit $q0 + ; CHECK-CSSC-LABEL: name: v4s32 + ; CHECK-CSSC: liveins: $q0 + ; CHECK-CSSC-NEXT: {{ $}} + ; CHECK-CSSC-NEXT: %val:_(<4 x s32>) = COPY $q0 + ; CHECK-CSSC-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1 + ; CHECK-CSSC-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32) + ; CHECK-CSSC-NEXT: [[XOR:%[0-9]+]]:_(<4 x s32>) = G_XOR %val, [[BUILD_VECTOR]] + ; CHECK-CSSC-NEXT: [[ADD:%[0-9]+]]:_(<4 x s32>) = G_ADD %val, [[BUILD_VECTOR]] + ; CHECK-CSSC-NEXT: [[AND:%[0-9]+]]:_(<4 x s32>) = G_AND [[XOR]], [[ADD]] + ; CHECK-CSSC-NEXT: [[CTPOP:%[0-9]+]]:_(<4 x s32>) = G_CTPOP [[AND]](<4 x s32>) + ; CHECK-CSSC-NEXT: $q0 = COPY [[CTPOP]](<4 x s32>) + ; CHECK-CSSC-NEXT: RET_ReallyLR implicit $q0 %val:_(<4 x s32>) = COPY $q0 %1:_(<4 x s32>) = G_CTTZ %val(<4 x s32>) $q0 = COPY %1(<4 x s32>) RET_ReallyLR implicit $q0 + ... diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-cttz.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-cttz.mir new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-cttz.mir @@ -0,0 +1,64 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -O0 -mtriple=arm64-unknown-unknown -global-isel -run-pass=instruction-select -global-isel-abort=1 %s -o - | FileCheck %s +# RUN: llc -O0 -mtriple=arm64-unknown-unknown -global-isel -mattr=+cssc -run-pass=instruction-select -global-isel-abort=1 %s -o - | FileCheck %s --check-prefix=CHECK-CSSC +... +--- +name: s32 +alignment: 4 +legalized: true +regBankSelected: true +tracksRegLiveness: true +body: | + bb.0: + liveins: $w0 + ; CHECK-LABEL: name: s32 + ; CHECK: liveins: $w0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32 = COPY $w0 + ; CHECK-NEXT: [[RBITWr:%[0-9]+]]:gpr32 = RBITWr [[COPY]] + ; CHECK-NEXT: [[CLZWr:%[0-9]+]]:gpr32 = CLZWr [[RBITWr]] + ; CHECK-NEXT: $w0 = COPY [[CLZWr]] + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + ; CHECK-CSSC-LABEL: name: s32 + ; CHECK-CSSC: liveins: $w0 + ; CHECK-CSSC-NEXT: {{ $}} + ; CHECK-CSSC-NEXT: [[COPY:%[0-9]+]]:gpr32 = COPY $w0 + ; CHECK-CSSC-NEXT: [[CTZWr:%[0-9]+]]:gpr32 = CTZWr [[COPY]] + ; CHECK-CSSC-NEXT: $w0 = COPY [[CTZWr]] + ; CHECK-CSSC-NEXT: RET_ReallyLR implicit $w0 + %0:gpr(s32) = COPY $w0 + %1:gpr(s32) = G_CTTZ %0(s32) + $w0 = COPY %1(s32) + RET_ReallyLR implicit $w0 + +... +--- +name: s64 +alignment: 4 +legalized: true +regBankSelected: true +tracksRegLiveness: true +body: | + bb.0: + liveins: $x0 + ; CHECK-LABEL: name: s64 + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[RBITXr:%[0-9]+]]:gpr64 = RBITXr [[COPY]] + ; CHECK-NEXT: [[CLZXr:%[0-9]+]]:gpr64 = CLZXr [[RBITXr]] + ; CHECK-NEXT: $x0 = COPY [[CLZXr]] + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + ; CHECK-CSSC-LABEL: name: s64 + ; CHECK-CSSC: liveins: $x0 + ; CHECK-CSSC-NEXT: {{ $}} + ; CHECK-CSSC-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-CSSC-NEXT: [[CTZXr:%[0-9]+]]:gpr64 = CTZXr [[COPY]] + ; CHECK-CSSC-NEXT: $x0 = COPY [[CTZXr]] + ; CHECK-CSSC-NEXT: RET_ReallyLR implicit $x0 + %0:gpr(s64) = COPY $x0 + %1:gpr(s64) = G_CTTZ %0(s64) + $x0 = COPY %1(s64) + RET_ReallyLR implicit $x0 + +...