diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -130,7 +130,8 @@ public: IntrinsicCostAttributes( Intrinsic::ID Id, const CallBase &CI, - InstructionCost ScalarCost = InstructionCost::getInvalid()); + InstructionCost ScalarCost = InstructionCost::getInvalid(), + bool TypeBasedOnly = false); IntrinsicCostAttributes( Intrinsic::ID Id, Type *RTy, ArrayRef Tys, diff --git a/llvm/lib/Analysis/CostModel.cpp b/llvm/lib/Analysis/CostModel.cpp --- a/llvm/lib/Analysis/CostModel.cpp +++ b/llvm/lib/Analysis/CostModel.cpp @@ -25,6 +25,7 @@ #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/IR/IntrinsicInst.h" using namespace llvm; static cl::opt CostKind( @@ -39,6 +40,9 @@ clEnumValN(TargetTransformInfo::TCK_SizeAndLatency, "size-latency", "Code size and latency"))); +static cl::opt TypeBasedIntrinsicCost("type-based-intrinsic-cost", + cl::desc("Calculate intrinsics cost based only on argument types"), + cl::init(false)); #define CM_NAME "cost-model" #define DEBUG_TYPE CM_NAME @@ -103,7 +107,16 @@ for (BasicBlock &B : *F) { for (Instruction &Inst : B) { - InstructionCost Cost = TTI->getInstructionCost(&Inst, CostKind); + InstructionCost Cost; + if (TypeBasedIntrinsicCost && isa(&Inst)) { + auto *II = dyn_cast(&Inst); + IntrinsicCostAttributes ICA(II->getIntrinsicID(), *II, + InstructionCost::getInvalid(), true); + Cost = TTI->getIntrinsicInstrCost(ICA, CostKind); + } + else { + Cost = TTI->getInstructionCost(&Inst, CostKind); + } if (auto CostVal = Cost.getValue()) OS << "Cost Model: Found an estimated cost of " << *CostVal; else @@ -122,7 +135,16 @@ for (Instruction &Inst : B) { // TODO: Use a pass parameter instead of cl::opt CostKind to determine // which cost kind to print. - InstructionCost Cost = TTI.getInstructionCost(&Inst, CostKind); + InstructionCost Cost; + if (TypeBasedIntrinsicCost && isa(&Inst)) { + auto *II = dyn_cast(&Inst); + IntrinsicCostAttributes ICA(II->getIntrinsicID(), *II, + InstructionCost::getInvalid(), true); + Cost = TTI.getIntrinsicInstrCost(ICA, CostKind); + } + else { + Cost = TTI.getInstructionCost(&Inst, CostKind); + } if (auto CostVal = Cost.getValue()) OS << "Cost Model: Found an estimated cost of " << *CostVal; else diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -58,14 +58,16 @@ } IntrinsicCostAttributes::IntrinsicCostAttributes( - Intrinsic::ID Id, const CallBase &CI, InstructionCost ScalarizationCost) + Intrinsic::ID Id, const CallBase &CI, InstructionCost ScalarizationCost, + bool TypeBasedOnly) : II(dyn_cast(&CI)), RetTy(CI.getType()), IID(Id), ScalarizationCost(ScalarizationCost) { if (const auto *FPMO = dyn_cast(&CI)) FMF = FPMO->getFastMathFlags(); - Arguments.insert(Arguments.begin(), CI.arg_begin(), CI.arg_end()); + if (!TypeBasedOnly) + Arguments.insert(Arguments.begin(), CI.arg_begin(), CI.arg_end()); FunctionType *FTy = CI.getCalledFunction()->getFunctionType(); ParamTys.insert(ParamTys.begin(), FTy->param_begin(), FTy->param_end()); } diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll --- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll +++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py ; RUN: opt < %s -passes='print' 2>&1 -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve | FileCheck %s +; RUN: opt < %s -passes='print' 2>&1 -type-based-intrinsic-cost -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve | FileCheck %s --check-prefix=TYPE_BASED_ONLY define void @vector_insert_extract( %v0, %v1, <16 x i32> %v2) { ; CHECK-LABEL: 'vector_insert_extract' @@ -8,6 +9,13 @@ ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %extract_scalable_from_scalable = call @llvm.vector.extract.nxv4i32.nxv16i32( %v1, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert_scalable_into_scalable = call @llvm.vector.insert.nxv16i32.nxv4i32( %v1, %v0, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'vector_insert_extract' +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %extract_fixed_from_scalable = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32( %v0, i64 0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %insert_fixed_into_scalable = call @llvm.vector.insert.nxv4i32.v16i32( %v0, <16 x i32> %v2, i64 0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %extract_scalable_from_scalable = call @llvm.vector.extract.nxv4i32.nxv16i32( %v1, i64 0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %insert_scalable_into_scalable = call @llvm.vector.insert.nxv16i32.nxv4i32( %v1, %v0, i64 0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %extract_fixed_from_scalable = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32( %v0, i64 0) %insert_fixed_into_scalable = call @llvm.vector.insert.nxv4i32.v16i32( %v0, <16 x i32> %v2, i64 0) @@ -48,6 +56,33 @@ ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %fmax_nxv4f32 = call fast float @llvm.vector.reduce.fmax.nxv4f32( %v2) ; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %fmax_nxv4f64 = call fast double @llvm.vector.reduce.fmax.nxv4f64( %v3) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'reductions' +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %add_nxv4i32 = call i32 @llvm.vector.reduce.add.nxv4i32( %v0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %add_nxv4i64 = call i64 @llvm.vector.reduce.add.nxv4i64( %v1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %mul_nxv4i32 = call i32 @llvm.vector.reduce.mul.nxv4i32( %v0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %mul_nxv4i64 = call i64 @llvm.vector.reduce.mul.nxv4i64( %v1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %and_nxv4i32 = call i32 @llvm.vector.reduce.and.nxv4i32( %v0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %and_nxv4i64 = call i64 @llvm.vector.reduce.and.nxv4i64( %v1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %or_nxv4i32 = call i32 @llvm.vector.reduce.or.nxv4i32( %v0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %or_nxv4i64 = call i64 @llvm.vector.reduce.or.nxv4i64( %v1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %xor_nxv4i32 = call i32 @llvm.vector.reduce.xor.nxv4i32( %v0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %xor_nxv4i64 = call i64 @llvm.vector.reduce.xor.nxv4i64( %v1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %umin_nxv4i32 = call i32 @llvm.vector.reduce.umin.nxv4i32( %v0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %umin_nxv4i64 = call i64 @llvm.vector.reduce.umin.nxv4i64( %v1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %smin_nxv4i32 = call i32 @llvm.vector.reduce.smin.nxv4i32( %v0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %smin_nxv4i64 = call i64 @llvm.vector.reduce.smin.nxv4i64( %v1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %umax_nxv4i32 = call i32 @llvm.vector.reduce.umax.nxv4i32( %v0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %umax_nxv4i64 = call i64 @llvm.vector.reduce.umax.nxv4i64( %v1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %smax_nxv4i32 = call i32 @llvm.vector.reduce.smax.nxv4i32( %v0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %smax_nxv4i64 = call i64 @llvm.vector.reduce.smax.nxv4i64( %v1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %fadd_nxv4f32 = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, %v2) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %fadd_nxv4f64 = call fast double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, %v3) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %fmin_nxv4f32 = call fast float @llvm.vector.reduce.fmin.nxv4f32( %v2) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %fmin_nxv4f64 = call fast double @llvm.vector.reduce.fmin.nxv4f64( %v3) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %fmax_nxv4f32 = call fast float @llvm.vector.reduce.fmax.nxv4f32( %v2) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %fmax_nxv4f64 = call fast double @llvm.vector.reduce.fmax.nxv4f64( %v3) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %add_nxv4i32 = call i32 @llvm.vector.reduce.add.nxv4i32( %v0) %add_nxv4i64 = call i64 @llvm.vector.reduce.add.nxv4i64( %v1) @@ -85,6 +120,13 @@ ; CHECK-NEXT: Cost Model: Invalid cost for instruction: %fmul_nxv4f32 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, %v0) ; CHECK-NEXT: Cost Model: Invalid cost for instruction: %fmul_nxv4f64 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, %v1) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'strict_fp_reductions' +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fadd_nxv4f32 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, %v0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %fadd_nxv4f64 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, %v1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %fmul_nxv4f32 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, %v0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %fmul_nxv4f64 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, %v1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %fadd_nxv4f32 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.0, %v0) %fadd_nxv4f64 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.0, %v1) @@ -127,6 +169,11 @@ ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %ctlz = call @llvm.ctlz.nxv4i32( %A, i1 true) ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cttz = call @llvm.cttz.nxv4i32( %A, i1 true) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'count_zeroes' +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %ctlz = call @llvm.ctlz.nxv4i32( %A, i1 true) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cttz = call @llvm.cttz.nxv4i32( %A, i1 true) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %ctlz = call @llvm.ctlz.nxv4i32( %A, i1 true) %cttz = call @llvm.cttz.nxv4i32( %A, i1 true) @@ -166,6 +213,36 @@ ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i1 = call @llvm.experimental.vector.reverse.nxv4i1( undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i1 = call @llvm.experimental.vector.reverse.nxv2i1( undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'vector_reverse' +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv16i8 = call @llvm.experimental.vector.reverse.nxv16i8( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv32i8 = call @llvm.experimental.vector.reverse.nxv32i8( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv2i16 = call @llvm.experimental.vector.reverse.nxv2i16( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv4i16 = call @llvm.experimental.vector.reverse.nxv4i16( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv8i16 = call @llvm.experimental.vector.reverse.nxv8i16( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv16i16 = call @llvm.experimental.vector.reverse.nxv16i16( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv4i32 = call @llvm.experimental.vector.reverse.nxv4i32( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv8i32 = call @llvm.experimental.vector.reverse.nxv8i32( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv2i64 = call @llvm.experimental.vector.reverse.nxv2i64( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv4i64 = call @llvm.experimental.vector.reverse.nxv4i64( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv2f16 = call @llvm.experimental.vector.reverse.nxv2f16( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv4f16 = call @llvm.experimental.vector.reverse.nxv4f16( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv8f16 = call @llvm.experimental.vector.reverse.nxv8f16( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv16f16 = call @llvm.experimental.vector.reverse.nxv16f16( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv2f32 = call @llvm.experimental.vector.reverse.nxv2f32( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv4f32 = call @llvm.experimental.vector.reverse.nxv4f32( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv8f32 = call @llvm.experimental.vector.reverse.nxv8f32( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv2f64 = call @llvm.experimental.vector.reverse.nxv2f64( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv4f64 = call @llvm.experimental.vector.reverse.nxv4f64( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv2bf16 = call @llvm.experimental.vector.reverse.nxv2bf16( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv4bf16 = call @llvm.experimental.vector.reverse.nxv4bf16( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv8bf16 = call @llvm.experimental.vector.reverse.nxv8bf16( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv16bf16 = call @llvm.experimental.vector.reverse.nxv16bf16( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv16i1 = call @llvm.experimental.vector.reverse.nxv16i1( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv8i1 = call @llvm.experimental.vector.reverse.nxv8i1( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv4i1 = call @llvm.experimental.vector.reverse.nxv4i1( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %reverse_nxv2i1 = call @llvm.experimental.vector.reverse.nxv2i1( undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %reverse_nxv16i8 = call @llvm.experimental.vector.reverse.nxv16i8( undef) @@ -237,6 +314,18 @@ ; CHECK-NEXT: Cost Model: Invalid cost for instruction: %log2 = call @llvm.log2.nxv4f32( %vec) ; CHECK-NEXT: Cost Model: Invalid cost for instruction: %log10 = call @llvm.log10.nxv4f32( %vec) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'unsupported_fp_ops' +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %sin = call @llvm.sin.nxv4f32( %vec) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %cos = call @llvm.cos.nxv4f32( %vec) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %pow = call @llvm.pow.nxv4f32( %vec, %vec) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %powi = call @llvm.powi.nxv4f32.i32( %vec, i32 %extraarg) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %exp = call @llvm.exp.nxv4f32( %vec) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %exp2 = call @llvm.exp2.nxv4f32( %vec) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %log = call @llvm.log.nxv4f32( %vec) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %log2 = call @llvm.log2.nxv4f32( %vec) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %log10 = call @llvm.log10.nxv4f32( %vec) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %sin = call @llvm.sin.nxv4f32( %vec) @@ -255,6 +344,10 @@ ; CHECK-LABEL: 'powi' ; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %powi = call @llvm.powi.nxv4f32.i32( %vec, i32 42) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'powi' +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %powi = call @llvm.powi.nxv4f32.i32( %vec, i32 42) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %powi = call @llvm.powi.nxv4f32.i32( %vec, i32 42) ret void @@ -327,6 +420,63 @@ ; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %splice_nxv4i1_neg = call @llvm.experimental.vector.splice.nxv4i1( zeroinitializer, zeroinitializer, i32 -1) ; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %splice_nxv2i1_neg = call @llvm.experimental.vector.splice.nxv2i1( zeroinitializer, zeroinitializer, i32 -1) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'vector_splice' +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv16i8 = call @llvm.experimental.vector.splice.nxv16i8( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv32i8 = call @llvm.experimental.vector.splice.nxv32i8( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv2i16 = call @llvm.experimental.vector.splice.nxv2i16( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4i16 = call @llvm.experimental.vector.splice.nxv4i16( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv8i16 = call @llvm.experimental.vector.splice.nxv8i16( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv16i16 = call @llvm.experimental.vector.splice.nxv16i16( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4i32 = call @llvm.experimental.vector.splice.nxv4i32( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv8i32 = call @llvm.experimental.vector.splice.nxv8i32( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv2i64 = call @llvm.experimental.vector.splice.nxv2i64( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4i64 = call @llvm.experimental.vector.splice.nxv4i64( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv2f16 = call @llvm.experimental.vector.splice.nxv2f16( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4f16 = call @llvm.experimental.vector.splice.nxv4f16( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv8f16 = call @llvm.experimental.vector.splice.nxv8f16( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv16f16 = call @llvm.experimental.vector.splice.nxv16f16( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv2f32 = call @llvm.experimental.vector.splice.nxv2f32( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4f32 = call @llvm.experimental.vector.splice.nxv4f32( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv8f32 = call @llvm.experimental.vector.splice.nxv8f32( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv2f64 = call @llvm.experimental.vector.splice.nxv2f64( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4f64 = call @llvm.experimental.vector.splice.nxv4f64( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv2bf16 = call @llvm.experimental.vector.splice.nxv2bf16( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4bf16 = call @llvm.experimental.vector.splice.nxv4bf16( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv8bf16 = call @llvm.experimental.vector.splice.nxv8bf16( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv16bf16 = call @llvm.experimental.vector.splice.nxv16bf16( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv16i1 = call @llvm.experimental.vector.splice.nxv16i1( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv8i1 = call @llvm.experimental.vector.splice.nxv8i1( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4i1 = call @llvm.experimental.vector.splice.nxv4i1( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv2i1 = call @llvm.experimental.vector.splice.nxv2i1( zeroinitializer, zeroinitializer, i32 1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv16i8_neg = call @llvm.experimental.vector.splice.nxv16i8( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv32i8_neg = call @llvm.experimental.vector.splice.nxv32i8( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv2i16_neg = call @llvm.experimental.vector.splice.nxv2i16( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4i16_neg = call @llvm.experimental.vector.splice.nxv4i16( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv8i16_neg = call @llvm.experimental.vector.splice.nxv8i16( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv16i16_neg = call @llvm.experimental.vector.splice.nxv16i16( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4i32_neg = call @llvm.experimental.vector.splice.nxv4i32( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv8i32_neg = call @llvm.experimental.vector.splice.nxv8i32( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv2i64_neg = call @llvm.experimental.vector.splice.nxv2i64( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4i64_neg = call @llvm.experimental.vector.splice.nxv4i64( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv2f16_neg = call @llvm.experimental.vector.splice.nxv2f16( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4f16_neg = call @llvm.experimental.vector.splice.nxv4f16( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv8f16_neg = call @llvm.experimental.vector.splice.nxv8f16( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv16f16_neg = call @llvm.experimental.vector.splice.nxv16f16( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv2f32_neg = call @llvm.experimental.vector.splice.nxv2f32( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4f32_neg = call @llvm.experimental.vector.splice.nxv4f32( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv8f32_neg = call @llvm.experimental.vector.splice.nxv8f32( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv2f64_neg = call @llvm.experimental.vector.splice.nxv2f64( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4f64_neg = call @llvm.experimental.vector.splice.nxv4f64( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv2bf16_neg = call @llvm.experimental.vector.splice.nxv2bf16( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4bf16_neg = call @llvm.experimental.vector.splice.nxv4bf16( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv8bf16_neg = call @llvm.experimental.vector.splice.nxv8bf16( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv16bf16_neg = call @llvm.experimental.vector.splice.nxv16bf16( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv16i1_neg = call @llvm.experimental.vector.splice.nxv16i1( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv8i1_neg = call @llvm.experimental.vector.splice.nxv8i1( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv4i1_neg = call @llvm.experimental.vector.splice.nxv4i1( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %splice_nxv2i1_neg = call @llvm.experimental.vector.splice.nxv2i1( zeroinitializer, zeroinitializer, i32 -1) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %splice_nxv16i8 = call @llvm.experimental.vector.splice.nxv16i8( zeroinitializer, zeroinitializer, i32 1) @@ -440,6 +590,29 @@ ; CHECK-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'get_lane_mask' +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %mask_nxv16i1_i64 = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %mask_nxv8i1_i64 = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 undef, i64 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %mask_nxv4i1_i64 = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 undef, i64 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %mask_nxv2i1_i64 = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 undef, i64 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %mask_nxv16i1_i32 = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 undef, i32 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %mask_nxv8i1_i32 = call @llvm.get.active.lane.mask.nxv8i1.i32(i32 undef, i32 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %mask_nxv4i1_i32 = call @llvm.get.active.lane.mask.nxv4i1.i32(i32 undef, i32 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %mask_nxv2i1_i32 = call @llvm.get.active.lane.mask.nxv2i1.i32(i32 undef, i32 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %mask_nxv32i1_i64 = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %mask_nxv16i1_i16 = call @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 61 for instruction: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %mask_v2i1_i64 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 undef, i64 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 61 for instruction: %mask_v16i1_i32 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 undef, i32 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 122 for instruction: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 61 for instruction: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %mask_nxv16i1_i64 = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef) %mask_nxv8i1_i64 = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 undef, i64 undef) @@ -477,6 +650,13 @@ ; CHECK-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %3 = call @llvm.fshr.nxv4i32( undef, undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %4 = call @llvm.fshr.nxv2i64( undef, undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'fshr' +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %1 = call @llvm.fshr.nxv16i8( undef, undef, undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %2 = call @llvm.fshr.nxv8i16( undef, undef, undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %3 = call @llvm.fshr.nxv4i32( undef, undef, undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %4 = call @llvm.fshr.nxv2i64( undef, undef, undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; call @llvm.fshr.nxv16i8( undef, undef, undef) call @llvm.fshr.nxv8i16( undef, undef, undef) @@ -492,6 +672,13 @@ ; CHECK-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %3 = call @llvm.fshl.nxv4i32( undef, undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %4 = call @llvm.fshl.nxv2i64( undef, undef, undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'fshl' +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %1 = call @llvm.fshl.nxv16i8( undef, undef, undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %2 = call @llvm.fshl.nxv8i16( undef, undef, undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %3 = call @llvm.fshl.nxv4i32( undef, undef, undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %4 = call @llvm.fshl.nxv2i64( undef, undef, undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; call @llvm.fshl.nxv16i8( undef, undef, undef) call @llvm.fshl.nxv8i16( undef, undef, undef) @@ -500,6 +687,114 @@ ret void } +define @masked_gather_nxv4i32( %ld, %masks, %passthru) { +; CHECK-LABEL: 'masked_gather_nxv4i32' +; CHECK-NEXT: Cost Model: Found an estimated cost of 80 for instruction: %res = call @llvm.masked.gather.nxv4i32.nxv4p0i32( %ld, i32 0, %masks, %passthru) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret %res +; +; TYPE_BASED_ONLY-LABEL: 'masked_gather_nxv4i32' +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %res = call @llvm.masked.gather.nxv4i32.nxv4p0i32( %ld, i32 0, %masks, %passthru) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret %res +; + %res = call @llvm.masked.gather.nxv4i32( %ld, i32 0, %masks, %passthru) + ret %res +} + +define @masked_gather_nxv8i32( %ld, %masks, %passthru) { +; CHECK-LABEL: 'masked_gather_nxv8i32' +; CHECK-NEXT: Cost Model: Found an estimated cost of 160 for instruction: %res = call @llvm.masked.gather.nxv8i32.nxv8p0i32( %ld, i32 0, %masks, %passthru) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret %res +; +; TYPE_BASED_ONLY-LABEL: 'masked_gather_nxv8i32' +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %res = call @llvm.masked.gather.nxv8i32.nxv8p0i32( %ld, i32 0, %masks, %passthru) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret %res +; + %res = call @llvm.masked.gather.nxv8i32( %ld, i32 0, %masks, %passthru) + ret %res +} + +define <4 x i32> @masked_gather_v4i32(<4 x i32*> %ld, <4 x i1> %masks, <4 x i32> %passthru) { +; CHECK-LABEL: 'masked_gather_v4i32' +; CHECK-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res +; +; TYPE_BASED_ONLY-LABEL: 'masked_gather_v4i32' +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 37 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res +; + %res = call <4 x i32> @llvm.masked.gather.v4i32(<4 x i32*> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru) + ret <4 x i32> %res +} + +define <1 x i128> @masked_gather_v1i128(<1 x i128*> %ld, <1 x i1> %masks, <1 x i128> %passthru) { +; CHECK-LABEL: 'masked_gather_v1i128' +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0i128(<1 x i128*> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <1 x i128> %res +; +; TYPE_BASED_ONLY-LABEL: 'masked_gather_v1i128' +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0i128(<1 x i128*> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <1 x i128> %res +; + %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0i128(<1 x i128*> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru) + ret <1 x i128> %res +} + +define void @masked_scatter_nxv4i32( %data, %ptrs, %masks) { +; CHECK-LABEL: 'masked_scatter_nxv4i32' +; CHECK-NEXT: Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0i32( %data, %ptrs, i32 0, %masks) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'masked_scatter_nxv4i32' +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0i32( %data, %ptrs, i32 0, %masks) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + + call void @llvm.masked.scatter.nxv4i32( %data, %ptrs, i32 0, %masks) + ret void +} + +define void @masked_scatter_nxv8i32( %data, %ptrs, %masks) { +; CHECK-LABEL: 'masked_scatter_nxv8i32' +; CHECK-NEXT: Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0i32( %data, %ptrs, i32 0, %masks) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'masked_scatter_nxv8i32' +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0i32( %data, %ptrs, i32 0, %masks) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + + call void @llvm.masked.scatter.nxv8i32( %data, %ptrs, i32 0, %masks) + ret void +} + +define void @masked_scatter_v4i32(<4 x i32> %data, <4 x i32*> %ptrs, <4 x i1> %masks) { +; CHECK-LABEL: 'masked_scatter_v4i32' +; CHECK-NEXT: Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %data, <4 x i32*> %ptrs, i32 0, <4 x i1> %masks) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'masked_scatter_v4i32' +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %data, <4 x i32*> %ptrs, i32 0, <4 x i1> %masks) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + + call void @llvm.masked.scatter.v4i32(<4 x i32> %data, <4 x i32*> %ptrs, i32 0, <4 x i1> %masks) + ret void +} + +define void @masked_scatter_v1i128(<1 x i128> %data, <1 x i128*> %ptrs, <1 x i1> %masks) { +; CHECK-LABEL: 'masked_scatter_v1i128' +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v1i128.v1p0i128(<1 x i128> %data, <1 x i128*> %ptrs, i32 0, <1 x i1> %masks) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; TYPE_BASED_ONLY-LABEL: 'masked_scatter_v1i128' +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v1i128.v1p0i128(<1 x i128> %data, <1 x i128*> %ptrs, i32 0, <1 x i1> %masks) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + + call void @llvm.masked.scatter.v1i128.v1p0i128(<1 x i128> %data, <1 x i128*> %ptrs, i32 0, <1 x i1> %masks) + ret void +} + declare @llvm.get.active.lane.mask.nxv16i1.i64(i64, i64) declare @llvm.get.active.lane.mask.nxv8i1.i64(i64, i64) declare @llvm.get.active.lane.mask.nxv4i1.i64(i64, i64) @@ -528,5 +823,13 @@ declare @llvm.fshl.nxv8i16(, , ) declare @llvm.fshl.nxv4i32(, , ) declare @llvm.fshl.nxv2i64(, , ) +declare @llvm.masked.gather.nxv4i32( %ptrs, i32 %align, %masks, %passthru) +declare @llvm.masked.gather.nxv8i32( %ptrs, i32 %align, %masks, %passthru) +declare <4 x i32> @llvm.masked.gather.v4i32(<4 x i32*> %ptrs, i32 %align, <4 x i1> %masks, <4 x i32> %passthru) +declare <1 x i128> @llvm.masked.gather.v1i128.v1p0i128(<1 x i128*>, i32, <1 x i1>, <1 x i128>) +declare void @llvm.masked.scatter.nxv4i32( %data, %ptrs, i32 %align, %masks) +declare void @llvm.masked.scatter.nxv8i32( %data, %ptrs, i32 %align, %masks) +declare void @llvm.masked.scatter.v4i32(<4 x i32> %data, <4 x i32*> %ptrs, i32 %align, <4 x i1> %masks) +declare void @llvm.masked.scatter.v1i128.v1p0i128(<1 x i128> %data, <1 x i128*> %ptrs, i32 %align, <1 x i1> %masks) attributes #0 = { "target-features"="+sve,+bf16" }