diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -344,6 +344,18 @@ setCondCodeAction(ISD::SETUGT, VT, Expand); setCondCodeAction(ISD::SETUEQ, VT, Expand); setCondCodeAction(ISD::SETUNE, VT, Expand); + + setOperationAction(ISD::FREM, VT, Expand); + setOperationAction(ISD::FPOW, VT, Expand); + setOperationAction(ISD::FPOWI, VT, Expand); + setOperationAction(ISD::FCOS, VT, Expand); + setOperationAction(ISD::FSIN, VT, Expand); + setOperationAction(ISD::FSINCOS, VT, Expand); + setOperationAction(ISD::FEXP, VT, Expand); + setOperationAction(ISD::FEXP2, VT, Expand); + setOperationAction(ISD::FLOG, VT, Expand); + setOperationAction(ISD::FLOG2, VT, Expand); + setOperationAction(ISD::FLOG10, VT, Expand); } } diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4973,8 +4973,8 @@ bool UseVectorIntrinsic = ID && IntrinsicCost <= CallCost; assert((UseVectorIntrinsic || !NeedToScalarize) && "Instruction should be scalarized elsewhere."); - assert(IntrinsicCost.isValid() && CallCost.isValid() && - "Cannot have invalid costs while widening"); + assert((IntrinsicCost.isValid() || CallCost.isValid()) && + "Either the intrinsic cost or vector call cost must be valid"); for (unsigned Part = 0; Part < UF; ++Part) { SmallVector Args; @@ -8538,8 +8538,8 @@ InstructionCost CallCost = CM.getVectorCallCost(CI, VF, NeedToScalarize); InstructionCost IntrinsicCost = ID ? CM.getVectorIntrinsicCost(CI, VF) : 0; bool UseVectorIntrinsic = ID && IntrinsicCost <= CallCost; - assert(IntrinsicCost.isValid() && CallCost.isValid() && - "Cannot have invalid costs while widening"); + assert((IntrinsicCost.isValid() || CallCost.isValid()) && + "Either the intrinsic cost or vector call cost must be valid"); return UseVectorIntrinsic || !NeedToScalarize; }; diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll --- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll +++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll @@ -181,4 +181,38 @@ declare @llvm.experimental.vector.reverse.nxv4i1() declare @llvm.experimental.vector.reverse.nxv2i1() +define void @unsupported_fp_ops( %vec) { +; CHECK-LABEL: 'unsupported_fp_ops' +; CHECK-NEXT: Cost Model: Invalid cost for instruction: %sin = call @llvm.sin.nxv4f32( %vec) +; CHECK-NEXT: Cost Model: Invalid cost for instruction: %cos = call @llvm.cos.nxv4f32( %vec) +; CHECK-NEXT: Cost Model: Invalid cost for instruction: %pow = call @llvm.pow.nxv4f32( %vec, %vec) +; CHECK-NEXT: Cost Model: Invalid cost for instruction: %powi = call @llvm.powi.nxv4f32( %vec, i32 42) +; CHECK-NEXT: Cost Model: Invalid cost for instruction: %exp = call @llvm.exp.nxv4f32( %vec) +; CHECK-NEXT: Cost Model: Invalid cost for instruction: %exp2 = call @llvm.exp2.nxv4f32( %vec) +; CHECK-NEXT: Cost Model: Invalid cost for instruction: %log = call @llvm.log.nxv4f32( %vec) +; CHECK-NEXT: Cost Model: Invalid cost for instruction: %log2 = call @llvm.log2.nxv4f32( %vec) +; CHECK-NEXT: Cost Model: Invalid cost for instruction: %log10 = call @llvm.log10.nxv4f32( %vec) + + %sin = call @llvm.sin.nxv4f32( %vec) + %cos = call @llvm.cos.nxv4f32( %vec) + %pow = call @llvm.pow.nxv4f32( %vec, %vec) + %powi = call @llvm.powi.nxv4f32( %vec, i32 42) + %exp = call @llvm.exp.nxv4f32( %vec) + %exp2 = call @llvm.exp2.nxv4f32( %vec) + %log = call @llvm.log.nxv4f32( %vec) + %log2 = call @llvm.log2.nxv4f32( %vec) + %log10 = call @llvm.log10.nxv4f32( %vec) + ret void +} + +declare @llvm.sin.nxv4f32() +declare @llvm.cos.nxv4f32() +declare @llvm.pow.nxv4f32(, ) +declare @llvm.powi.nxv4f32(, i32) +declare @llvm.exp.nxv4f32() +declare @llvm.exp2.nxv4f32() +declare @llvm.log.nxv4f32() +declare @llvm.log2.nxv4f32() +declare @llvm.log10.nxv4f32() + attributes #0 = { "target-features"="+sve,+bf16" } diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll @@ -72,11 +72,10 @@ } define void @vec_intrinsic(i64 %N, double* nocapture readonly %a) { -;; FIXME: Should be calling sin_vec, once the cost of scalarizing is handled. ; CHECK-LABEL: @vec_intrinsic ; CHECK: vector.body: ; CHECK: %[[LOAD:.*]] = load , * -; CHECK: call fast @llvm.sin.nxv2f64( %[[LOAD]]) +; CHECK: call fast @sin_vec( %[[LOAD]]) entry: %cmp7 = icmp sgt i64 %N, 0 br i1 %cmp7, label %for.body, label %for.end