diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h --- a/llvm/lib/Target/AArch64/AArch64Subtarget.h +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h @@ -209,9 +209,7 @@ } unsigned getMaxInterleaveFactor() const { return MaxInterleaveFactor; } - unsigned getVectorInsertExtractBaseCost() const { - return VectorInsertExtractBaseCost; - } + unsigned getVectorInsertExtractBaseCost() const; unsigned getCacheLineSize() const override { return CacheLineSize; } unsigned getPrefetchDistance() const override { return PrefetchDistance; } unsigned getMinPrefetchStride(unsigned NumMemAccesses, diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp --- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp @@ -52,6 +52,16 @@ static cl::opt UseAA("aarch64-use-aa", cl::init(true), cl::desc("Enable the use of AA during codegen.")); +static cl::opt OverrideVectorInsertExtractBaseCost( + "aarch64-insert-extract-base-cost", + cl::desc("Base cost of vector insert/extract element"), cl::Hidden); + +unsigned AArch64Subtarget::getVectorInsertExtractBaseCost() const { + if (OverrideVectorInsertExtractBaseCost.getNumOccurrences() > 0) + return OverrideVectorInsertExtractBaseCost; + return VectorInsertExtractBaseCost; +} + AArch64Subtarget &AArch64Subtarget::initializeSubtargetDependencies( StringRef FS, StringRef CPUString, StringRef TuneCPUString) { // Determine default and user-specified characteristics diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-insert-extract.ll b/llvm/test/Analysis/CostModel/AArch64/sve-insert-extract.ll --- a/llvm/test/Analysis/CostModel/AArch64/sve-insert-extract.ll +++ b/llvm/test/Analysis/CostModel/AArch64/sve-insert-extract.ll @@ -1,57 +1,203 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='print' 2>&1 -disable-output -S < %s | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py + +; RUN: opt -passes='print' 2>&1 -disable-output -S < %s | FileCheck --check-prefix=CHECK-DEFAULT %s +; RUN: opt -aarch64-insert-extract-base-cost=0 -passes='print' 2>&1 -disable-output -S < %s | FileCheck --check-prefix=CHECK-LOW %s +; RUN: opt -aarch64-insert-extract-base-cost=100000 -passes='print' 2>&1 -disable-output -S < %s | FileCheck --check-prefix=CHECK-HIGH %s target triple = "aarch64-unknown-linux-gnu" define void @ins_el0() #0 { -; CHECK: Cost Model: Found an estimated cost of 0 for instruction: %v0 = insertelement zeroinitializer, i8 0, i64 0 -; CHECK: Cost Model: Found an estimated cost of 0 for instruction: %v1 = insertelement zeroinitializer, i16 0, i64 0 -; CHECK: Cost Model: Found an estimated cost of 0 for instruction: %v2 = insertelement zeroinitializer, i32 0, i64 0 -; CHECK: Cost Model: Found an estimated cost of 0 for instruction: %v3 = insertelement zeroinitializer, i64 0, i64 0 +; CHECK-DEFAULT-LABEL: 'ins_el0' +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v0 = insertelement zeroinitializer, i8 0, i64 0 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v1 = insertelement zeroinitializer, i16 0, i64 0 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2 = insertelement zeroinitializer, i32 0, i64 0 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v3 = insertelement zeroinitializer, i64 0, i64 0 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v4 = insertelement zeroinitializer, float 0.000000e+00, i64 0 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v5 = insertelement zeroinitializer, double 0.000000e+00, i64 0 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CHECK-LOW-LABEL: 'ins_el0' +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v0 = insertelement zeroinitializer, i8 0, i64 0 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v1 = insertelement zeroinitializer, i16 0, i64 0 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2 = insertelement zeroinitializer, i32 0, i64 0 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v3 = insertelement zeroinitializer, i64 0, i64 0 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v4 = insertelement zeroinitializer, float 0.000000e+00, i64 0 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v5 = insertelement zeroinitializer, double 0.000000e+00, i64 0 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CHECK-HIGH-LABEL: 'ins_el0' +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v0 = insertelement zeroinitializer, i8 0, i64 0 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v1 = insertelement zeroinitializer, i16 0, i64 0 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2 = insertelement zeroinitializer, i32 0, i64 0 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v3 = insertelement zeroinitializer, i64 0, i64 0 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v4 = insertelement zeroinitializer, float 0.000000e+00, i64 0 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v5 = insertelement zeroinitializer, double 0.000000e+00, i64 0 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; %v0 = insertelement zeroinitializer, i8 0, i64 0 %v1 = insertelement zeroinitializer, i16 0, i64 0 %v2 = insertelement zeroinitializer, i32 0, i64 0 %v3 = insertelement zeroinitializer, i64 0, i64 0 + %v4 = insertelement zeroinitializer, float 0., i64 0 + %v5 = insertelement zeroinitializer, double 0., i64 0 ret void } define void @ins_el1() #0 { -; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %v0 = insertelement zeroinitializer, i8 0, i64 1 -; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %v1 = insertelement zeroinitializer, i16 0, i64 1 -; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %v2 = insertelement zeroinitializer, i32 0, i64 1 -; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %v3 = insertelement zeroinitializer, i64 0, i64 1 +; CHECK-DEFAULT-LABEL: 'ins_el1' +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v0 = insertelement zeroinitializer, i8 0, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v1 = insertelement zeroinitializer, i16 0, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2 = insertelement zeroinitializer, i32 0, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v3 = insertelement zeroinitializer, i64 0, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4 = insertelement zeroinitializer, float 0.000000e+00, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v5 = insertelement zeroinitializer, double 0.000000e+00, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CHECK-LOW-LABEL: 'ins_el1' +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v0 = insertelement zeroinitializer, i8 0, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v1 = insertelement zeroinitializer, i16 0, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2 = insertelement zeroinitializer, i32 0, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v3 = insertelement zeroinitializer, i64 0, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v4 = insertelement zeroinitializer, float 0.000000e+00, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v5 = insertelement zeroinitializer, double 0.000000e+00, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CHECK-HIGH-LABEL: 'ins_el1' +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v0 = insertelement zeroinitializer, i8 0, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v1 = insertelement zeroinitializer, i16 0, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v2 = insertelement zeroinitializer, i32 0, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v3 = insertelement zeroinitializer, i64 0, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v4 = insertelement zeroinitializer, float 0.000000e+00, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v5 = insertelement zeroinitializer, double 0.000000e+00, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; %v0 = insertelement zeroinitializer, i8 0, i64 1 %v1 = insertelement zeroinitializer, i16 0, i64 1 %v2 = insertelement zeroinitializer, i32 0, i64 1 %v3 = insertelement zeroinitializer, i64 0, i64 1 + %v4 = insertelement zeroinitializer, float 0., i64 1 + %v5 = insertelement zeroinitializer, double 0., i64 1 ret void } define void @ext_el0() #0 { -; CHECK: Cost Model: Found an estimated cost of 0 for instruction: %v0 = extractelement zeroinitializer, i64 0 -; CHECK: Cost Model: Found an estimated cost of 0 for instruction: %v1 = extractelement zeroinitializer, i64 0 -; CHECK: Cost Model: Found an estimated cost of 0 for instruction: %v2 = extractelement zeroinitializer, i64 0 -; CHECK: Cost Model: Found an estimated cost of 0 for instruction: %v3 = extractelement zeroinitializer, i64 0 +; CHECK-DEFAULT-LABEL: 'ext_el0' +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v0 = extractelement zeroinitializer, i64 0 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v1 = extractelement zeroinitializer, i64 0 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2 = extractelement zeroinitializer, i64 0 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v3 = extractelement zeroinitializer, i64 0 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v4 = extractelement zeroinitializer, i64 0 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v5 = extractelement zeroinitializer, i64 0 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CHECK-LOW-LABEL: 'ext_el0' +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v0 = extractelement zeroinitializer, i64 0 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v1 = extractelement zeroinitializer, i64 0 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2 = extractelement zeroinitializer, i64 0 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v3 = extractelement zeroinitializer, i64 0 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v4 = extractelement zeroinitializer, i64 0 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v5 = extractelement zeroinitializer, i64 0 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CHECK-HIGH-LABEL: 'ext_el0' +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v0 = extractelement zeroinitializer, i64 0 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v1 = extractelement zeroinitializer, i64 0 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2 = extractelement zeroinitializer, i64 0 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v3 = extractelement zeroinitializer, i64 0 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v4 = extractelement zeroinitializer, i64 0 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v5 = extractelement zeroinitializer, i64 0 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; %v0 = extractelement zeroinitializer, i64 0 %v1 = extractelement zeroinitializer, i64 0 %v2 = extractelement zeroinitializer, i64 0 %v3 = extractelement zeroinitializer, i64 0 + %v4 = extractelement zeroinitializer, i64 0 + %v5 = extractelement zeroinitializer, i64 0 ret void } define void @ext_el1() #0 { -; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %v0 = extractelement zeroinitializer, i64 1 -; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %v1 = extractelement zeroinitializer, i64 1 -; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %v2 = extractelement zeroinitializer, i64 1 -; CHECK: Cost Model: Found an estimated cost of 3 for instruction: %v3 = extractelement zeroinitializer, i64 1 +; CHECK-DEFAULT-LABEL: 'ext_el1' +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v0 = extractelement zeroinitializer, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v1 = extractelement zeroinitializer, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2 = extractelement zeroinitializer, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v3 = extractelement zeroinitializer, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4 = extractelement zeroinitializer, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v5 = extractelement zeroinitializer, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CHECK-LOW-LABEL: 'ext_el1' +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v0 = extractelement zeroinitializer, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v1 = extractelement zeroinitializer, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2 = extractelement zeroinitializer, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v3 = extractelement zeroinitializer, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v4 = extractelement zeroinitializer, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v5 = extractelement zeroinitializer, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CHECK-HIGH-LABEL: 'ext_el1' +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v0 = extractelement zeroinitializer, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v1 = extractelement zeroinitializer, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v2 = extractelement zeroinitializer, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v3 = extractelement zeroinitializer, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v4 = extractelement zeroinitializer, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v5 = extractelement zeroinitializer, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %v0 = extractelement zeroinitializer, i64 1 + %v1 = extractelement zeroinitializer, i64 1 + %v2 = extractelement zeroinitializer, i64 1 + %v3 = extractelement zeroinitializer, i64 1 + %v4 = extractelement zeroinitializer, i64 1 + %v5 = extractelement zeroinitializer, i64 1 + ret void +} + + +; Test the behaviour in the presence of a CPU-specific override in AArch64Subtarget (via attribute set). +define void @test_override_cpu_given() #1 { +; CHECK-DEFAULT-LABEL: 'test_override_cpu_given' +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v0 = extractelement zeroinitializer, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v1 = extractelement zeroinitializer, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2 = extractelement zeroinitializer, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v3 = extractelement zeroinitializer, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4 = extractelement zeroinitializer, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v5 = extractelement zeroinitializer, i64 1 +; CHECK-DEFAULT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CHECK-LOW-LABEL: 'test_override_cpu_given' +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v0 = extractelement zeroinitializer, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v1 = extractelement zeroinitializer, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2 = extractelement zeroinitializer, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v3 = extractelement zeroinitializer, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v4 = extractelement zeroinitializer, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v5 = extractelement zeroinitializer, i64 1 +; CHECK-LOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CHECK-HIGH-LABEL: 'test_override_cpu_given' +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v0 = extractelement zeroinitializer, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v1 = extractelement zeroinitializer, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v2 = extractelement zeroinitializer, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v3 = extractelement zeroinitializer, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v4 = extractelement zeroinitializer, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 100000 for instruction: %v5 = extractelement zeroinitializer, i64 1 +; CHECK-HIGH-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; %v0 = extractelement zeroinitializer, i64 1 %v1 = extractelement zeroinitializer, i64 1 %v2 = extractelement zeroinitializer, i64 1 %v3 = extractelement zeroinitializer, i64 1 + %v4 = extractelement zeroinitializer, i64 1 + %v5 = extractelement zeroinitializer, i64 1 ret void } + + + attributes #0 = { "target-features"="+sve" vscale_range(1, 16) } +attributes #1 = { "target-features"="+sve" vscale_range(1, 16) "target-cpu"="kryo" }