diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -1346,15 +1346,12 @@ return Cost; } } - // TODO: Handle the remaining intrinsic with scalable vector type - if (isa(RetTy)) - return BaseT::getIntrinsicInstrCost(ICA, CostKind); // Assume that we need to scalarize this intrinsic. // Compute the scalarization overhead based on Args for a vector // intrinsic. unsigned ScalarizationCost = std::numeric_limits::max(); - if (RetVF.isVector()) { + if (RetVF.isVector() && !RetVF.isScalable()) { ScalarizationCost = 0; if (!RetTy->isVoidTy()) ScalarizationCost += @@ -1399,6 +1396,12 @@ SmallVector ISDs; switch (IID) { default: { + // Scalable vectors cannot be scalarized, so return Invalid. + if (isa(RetTy) || any_of(Tys, [](const Type *Ty) { + return isa(Ty); + })) + return InstructionCost::getInvalid(); + // Assume that we need to scalarize this intrinsic. InstructionCost ScalarizationCost = ScalarizationCostPassed; unsigned ScalarCalls = 1; @@ -1810,6 +1813,12 @@ // this will emit a costly libcall, adding call overhead and spills. Make it // very expensive. if (auto *RetVTy = dyn_cast(RetTy)) { + // Scalable vectors cannot be scalarized, so return Invalid. + if (isa(RetTy) || any_of(Tys, [](const Type *Ty) { + return isa(Ty); + })) + return InstructionCost::getInvalid(); + unsigned ScalarizationCost = SkipScalarizationCost ? ScalarizationCostPassed : getScalarizationOverhead(RetVTy, true, false); diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll --- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll +++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll @@ -105,8 +105,8 @@ define void @count_zeroes( %A) { ; CHECK-LABEL: 'count_zeroes' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %ctlz = call @llvm.ctlz.nxv4i32( %A, i1 true) -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call @llvm.cttz.nxv4i32( %A, i1 true) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %ctlz = call @llvm.ctlz.nxv4i32( %A, i1 true) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cttz = call @llvm.cttz.nxv4i32( %A, i1 true) %ctlz = call @llvm.ctlz.nxv4i32( %A, i1 true) %cttz = call @llvm.cttz.nxv4i32( %A, i1 true) ret void diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-math.ll b/llvm/test/Analysis/CostModel/AArch64/sve-math.ll --- a/llvm/test/Analysis/CostModel/AArch64/sve-math.ll +++ b/llvm/test/Analysis/CostModel/AArch64/sve-math.ll @@ -29,7 +29,7 @@ define @sqrt_v2f64( %a) { ; THRU-LABEL: 'sqrt_v2f64' -; THRU-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %r = call @llvm.sqrt.nxv2f64( %a) +; THRU-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %r = call @llvm.sqrt.nxv2f64( %a) ; THRU-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret %r ; ; LATE-LABEL: 'sqrt_v2f64' @@ -37,11 +37,11 @@ ; LATE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret %r ; ; SIZE-LABEL: 'sqrt_v2f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %r = call @llvm.sqrt.nxv2f64( %a) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %r = call @llvm.sqrt.nxv2f64( %a) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret %r ; ; SIZE_LATE-LABEL: 'sqrt_v2f64' -; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %r = call @llvm.sqrt.nxv2f64( %a) +; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %r = call @llvm.sqrt.nxv2f64( %a) ; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret %r ; %r = call @llvm.sqrt.v2f64( %a)