diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp --- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp @@ -226,6 +226,7 @@ PrefLoopAlignment = Align(32); MaxBytesForLoopAlignment = 16; VScaleForTuning = 2; + DefaultSVETFOpts = TailFoldingOpts::Simple; break; case Neoverse512TVB: PrefFunctionAlignment = Align(16); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -39,6 +39,9 @@ static cl::opt SVEScatterOverhead("sve-scatter-overhead", cl::init(10), cl::Hidden); +static cl::opt SVETailFoldInsnThreshold("sve-tail-folding-insn-threshold", + cl::init(15), cl::Hidden); + namespace { class TailFoldingOption { // These bitfields will only ever be set to something non-zero in operator=, @@ -3558,8 +3561,19 @@ if (Required == TailFoldingOpts::Disabled) Required |= TailFoldingOpts::Simple; - return TailFoldingOptionLoc.satisfies(ST->getSVETailFoldingDefaultOpts(), - Required); + if (!TailFoldingOptionLoc.satisfies(ST->getSVETailFoldingDefaultOpts(), + Required)) + return false; + + // Don't tail-fold for tight loops where we would be better off interleaving + // with an unpredicated loop. + unsigned NumInsns = 0; + for (BasicBlock *BB : TFI->LVL->getLoop()->blocks()) { + NumInsns += BB->sizeWithoutDebug(); + } + + // We expect 4 of these to be a IV PHI, IV add, IV compare and branch. + return NumInsns >= SVETailFoldInsnThreshold; } InstructionCost diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll @@ -1,7 +1,5 @@ ; RUN: opt -S -passes=loop-vectorize,instsimplify -force-vector-interleave=1 \ -; RUN: -mcpu=neoverse-v1 < %s | FileCheck %s --check-prefix=CHECK-EPILOG -; RUN: opt -S -passes=loop-vectorize,instsimplify -force-vector-interleave=1 \ -; RUN: -mcpu=neoverse-v1 < %s | FileCheck %s --check-prefix=CHECK-EPILOG +; RUN: -mcpu=neoverse-v1 -sve-tail-folding=disabled < %s | FileCheck %s --check-prefix=CHECK-EPILOG ; RUN: opt -S -passes=loop-vectorize,instsimplify -force-vector-interleave=1 \ ; RUN: -mcpu=neoverse-v2 < %s | FileCheck %s --check-prefix=CHECK-NO-EPILOG ; RUN: opt -S -passes=loop-vectorize,instsimplify -force-vector-interleave=1 \ diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll @@ -1,11 +1,16 @@ -; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=disabled -S | FileCheck %s -check-prefix=CHECK-NOTF -; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=default -S | FileCheck %s -check-prefix=CHECK-NOTF -; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=all -S | FileCheck %s -check-prefix=CHECK-TF -; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=simple+reductions+recurrences+reverse -S | FileCheck %s -check-prefix=CHECK-TF -; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=all+noreductions -S | FileCheck %s -check-prefix=CHECK-TF-NORED -; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=all+norecurrences -S | FileCheck %s -check-prefix=CHECK-TF-NOREC -; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=all+noreverse -S | FileCheck %s -check-prefix=CHECK-TF-NOREV -; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=reductions -S | FileCheck %s -check-prefix=CHECK-TF-ONLYRED +; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=disabled -S | FileCheck %s -check-prefix=CHECK-NOTF +; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=default -S | FileCheck %s -check-prefix=CHECK-NOTF +; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S | FileCheck %s -check-prefix=CHECK-NOTF +; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all -S | FileCheck %s -check-prefix=CHECK-TF +; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=simple+reductions+recurrences+reverse -S | FileCheck %s -check-prefix=CHECK-TF +; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S -mcpu=neoverse-v1 -sve-tail-folding=default+reductions+recurrences+reverse | FileCheck %s -check-prefix=CHECK-TF +; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all+noreductions -S | FileCheck %s -check-prefix=CHECK-TF-NORED +; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all+norecurrences -S | FileCheck %s -check-prefix=CHECK-TF-NOREC +; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all+noreverse -S | FileCheck %s -check-prefix=CHECK-TF-NOREV +; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=reductions -S | FileCheck %s -check-prefix=CHECK-TF-ONLYRED +; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S -sve-tail-folding=default -mcpu=neoverse-v1 | FileCheck %s -check-prefix=CHECK-NEOVERSE-V1 +; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S -mcpu=neoverse-v1 -sve-tail-folding=default | FileCheck %s -check-prefix=CHECK-NEOVERSE-V1 +; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S -mcpu=neoverse-v1 | FileCheck %s -check-prefix=CHECK-NEOVERSE-V1 target triple = "aarch64-unknown-linux-gnu" @@ -58,6 +63,14 @@ ; CHECK-TF-ONLYRED-NOT: %{{.*}} = phi ; CHECK-TF-ONLYRED: store %[[SPLAT]], ptr +; CHECK-NEOVERSE-V1-LABEL: @simple_memset( +; CHECK-NEOVERSE-V1: vector.ph: +; CHECK-NEOVERSE-V1: %[[INSERT:.*]] = insertelement poison, i32 %val, i64 0 +; CHECK-NEOVERSE-V1: %[[SPLAT:.*]] = shufflevector %[[INSERT]], poison, zeroinitializer +; CHECK-NEOVERSE-V1: vector.body: +; CHECK-NEOVERSE-V1: %[[ACTIVE_LANE_MASK:.*]] = phi +; CHECK-NEOVERSE-V1: call void @llvm.masked.store.nxv4i32.p0( %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]] + entry: br label %while.body @@ -129,6 +142,15 @@ ; CHECK-TF-ONLYRED: %[[SEL:.*]] = select fast %[[ACTIVE_LANE_MASK]], %[[ADD]], %[[VEC_PHI]] ; CHECK-TF-ONLYRED: middle.block: ; CHECK-TF-ONLYRED-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, %[[SEL]]) + +; CHECK-NEOVERSE-V1-LABEL: @fadd_red_fast +; CHECK-NEOVERSE-V1: vector.body: +; CHECK-NEOVERSE-V1-NOT: %{{.*}} = phi +; CHECK-NEOVERSE-V1: %[[LOAD:.*]] = load +; CHECK-NEOVERSE-V1: %[[ADD:.*]] = fadd fast %[[LOAD]] +; CHECK-NEOVERSE-V1: middle.block: +; CHECK-NEOVERSE-V1-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, %[[ADD]]) + entry: br label %for.body @@ -225,6 +247,19 @@ ; CHECK-TF-ONLYRED: %[[ADD:.*]] = add nsw %[[LOAD]], %[[SPLICE]] ; CHECK-TF-ONLYRED: store %[[ADD]] +; CHECK-NEOVERSE-V1-LABEL: @add_recur +; CHECK-NEOVERSE-V1: entry: +; CHECK-NEOVERSE-V1: %[[PRE:.*]] = load i32, ptr %src, align 4 +; CHECK-NEOVERSE-V1: vector.ph: +; CHECK-NEOVERSE-V1: %[[RECUR_INIT:.*]] = insertelement poison, i32 %[[PRE]] +; CHECK-NEOVERSE-V1: vector.body: +; CHECK-NEOVERSE-V1-NOT: %{{.*}} = phi +; CHECK-NEOVERSE-V1: %[[VECTOR_RECUR:.*]] = phi [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ] +; CHECK-NEOVERSE-V1: %[[LOAD]] = load +; CHECK-NEOVERSE-V1: %[[SPLICE:.*]] = call @llvm.experimental.vector.splice.nxv4i32( %[[VECTOR_RECUR]], %[[LOAD]], i32 -1) +; CHECK-NEOVERSE-V1: %[[ADD:.*]] = add nsw %[[LOAD]], %[[SPLICE]] +; CHECK-NEOVERSE-V1: store %[[ADD]] + entry: %.pre = load i32, ptr %src, align 4 br label %for.body @@ -276,6 +311,12 @@ ; CHECK-TF-NOREV: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> ; CHECK-TF-NOREV: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> +; CHECK-NEOVERSE-V1-LABEL: @interleave( +; CHECK-NEOVERSE-V1: vector.body: +; CHECK-NEOVERSE-V1: %[[LOAD:.*]] = load <8 x float>, ptr +; CHECK-NEOVERSE-V1: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> +; CHECK-NEOVERSE-V1: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> + entry: br label %for.body @@ -335,6 +376,12 @@ ; CHECK-TF-NOREC: %[[REVERSE_MASK:.*]] = call @llvm.experimental.vector.reverse.nxv2i1( %[[ACTIVE_LANE_MASK]]) ; CHECK-TF-NOREC: %[[MASKED_LOAD:.*]] = call @llvm.masked.load.nxv2f64.p0({{.*}} %reverse +; CHECK-TF-NEOVERSE-V1-LABEL: @reverse( +; CHECK-TF-NEOVERSE-V1: vector.body: +; CHECK-TF-NEOVERSE-V1-NOT: %{{.*}} = phi +; CHECK-TF-NEOVERSE-V1: %[[LOAD:.*]] = load , * %18, align 8 +; CHECK-TF-NEOVERSE-V1: %{{.*}} = call @llvm.experimental.vector.reverse.nxv2f64( %[[LOAD]]) + entry: br label %for.body diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-overflow-checks.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-overflow-checks.ll --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-overflow-checks.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-overflow-checks.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='loop-vectorize,instcombine' -sve-tail-folding=all -S < %s | FileCheck %s +; RUN: opt -passes='loop-vectorize,instcombine' -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all -S < %s | FileCheck %s target triple = "aarch64"