Index: llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -707,7 +707,7 @@ int ISD = TLI->InstructionOpcodeToISD(Opcode); // We don't lower some vector selects well that are wider than the register // width. - if (ValTy->isVectorTy() && ISD == ISD::SELECT) { + if (isa(ValTy) && ISD == ISD::SELECT) { // We would need this many instructions to hide the scalarization happening. const int AmortizationCost = 20; @@ -749,6 +749,8 @@ return Entry->Cost; } } + // The base case handles scalable vectors fine for now, since it treats the + // cost as 1 * legalization cost. return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); } Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7201,10 +7201,8 @@ const SCEV *CondSCEV = SE->getSCEV(SI->getCondition()); bool ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop)); Type *CondTy = SI->getCondition()->getType(); - if (!ScalarCond) { - assert(!VF.isScalable() && "VF is assumed to be non scalable."); + if (!ScalarCond) CondTy = VectorType::get(CondTy, VF); - } return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, CondTy, CmpInst::BAD_ICMP_PREDICATE, CostKind, I); } Index: llvm/test/Analysis/CostModel/sve-cmpsel.ll =================================================================== --- /dev/null +++ llvm/test/Analysis/CostModel/sve-cmpsel.ll @@ -0,0 +1,260 @@ +; RUN: opt -cost-model -analyze -mtriple=aarch64--linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s + +; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t + +; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it. +; WARN-NOT: warning + +; Check icmp for a legal integer vector. +define @cmp_nxv2i64() { +; CHECK-LABEL: 'cmp_nxv2i64' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = icmp ne undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = icmp ne undef, undef + ret %res +} + +define @cmp_nxv4i32() { +; CHECK-LABEL: 'cmp_nxv4i32' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = icmp ne undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = icmp ne undef, undef + ret %res +} + +define @cmp_nxv8i16() { +; CHECK-LABEL: 'cmp_nxv8i16' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = icmp ne undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = icmp ne undef, undef + ret %res +} + +define @cmp_nxv16i8() { +; CHECK-LABEL: 'cmp_nxv16i8' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = icmp ne undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = icmp ne undef, undef + ret %res +} + +; Check icmp for an illegal integer vector. +define @cmp_nxv4i64() { +; CHECK-LABEL: 'cmp_nxv4i64' +; CHECK: Cost Model: Found an estimated cost of 2 for instruction: %res = icmp ne undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = icmp ne undef, undef + ret %res +} + +; Check icmp for predicate vectors. +define @cmp_nxv2i1() { +; CHECK-LABEL: 'cmp_nxv2i1' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = icmp ne undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = icmp ne undef, undef + ret %res +} + +define @cmp_nxv4i1() { +; CHECK-LABEL: 'cmp_nxv4i1' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = icmp ne undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = icmp ne undef, undef + ret %res +} + +define @cmp_nxv8i1() { +; CHECK-LABEL: 'cmp_nxv8i1' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = icmp ne undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = icmp ne undef, undef + ret %res +} + +define @cmp_nxv16i1() { +; CHECK-LABEL: 'cmp_nxv16i1' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = icmp ne undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = icmp ne undef, undef + ret %res +} + +; Check icmp for an illegal predicate vector. +define @cmp_nxv32i1() { +; CHECK-LABEL: 'cmp_nxv32i1' +; CHECK: Cost Model: Found an estimated cost of 2 for instruction: %res = icmp ne undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = icmp ne undef, undef + ret %res +} + +; Check fcmp for a legal FP vector +define @cmp_nxv2f64() { +; CHECK-LABEL: 'cmp_nxv2f64' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = fcmp oge undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = fcmp oge undef, undef + ret %res +} + +define @cmp_nxv4f32() { +; CHECK-LABEL: 'cmp_nxv4f32' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = fcmp oge undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = fcmp oge undef, undef + ret %res +} + +define @cmp_nxv8f16() { +; CHECK-LABEL: 'cmp_nxv8f16' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = fcmp oge undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = fcmp oge undef, undef + ret %res +} + +define @cmp_nxv8bf16() #0 { +; CHECK-LABEL: 'cmp_nxv8bf16' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = fcmp oge undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = fcmp oge undef, undef + ret %res +} + +; Check fcmp for an illegal FP vector +define @cmp_nxv16f16() { +; CHECK-LABEL: 'cmp_nxv16f16' +; CHECK: Cost Model: Found an estimated cost of 2 for instruction: %res = fcmp oge undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = fcmp oge undef, undef + ret %res +} + +; Check select for a legal integer vector +define @sel_nxv2i64() { +; CHECK-LABEL: 'sel_nxv2i64' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +define @sel_nxv4i32() { +; CHECK-LABEL: 'sel_nxv4i32' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +define @sel_nxv8i16() { +; CHECK-LABEL: 'sel_nxv8i16' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +define @sel_nxv16i8() { +; CHECK-LABEL: 'sel_nxv16i8' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +; Check select for an illegal integer vector +define @sel_nxv16i16() { +; CHECK-LABEL: 'sel_nxv16i16' +; CHECK: Cost Model: Found an estimated cost of 2 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +; Check select for a legal FP vector +define @sel_nxv2f64() { +; CHECK-LABEL: 'sel_nxv2f64' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +define @sel_nxv4f32() { +; CHECK-LABEL: 'sel_nxv4f32' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +define @sel_nxv8f16() { +; CHECK-LABEL: 'sel_nxv8f16' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +define @sel_nxv8bf16() #0 { +; CHECK-LABEL: 'sel_nxv8bf16' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +; Check select for an illegal FP vector +define @sel_nxv8f32() { +; CHECK-LABEL: 'sel_nxv8f32' +; CHECK: Cost Model: Found an estimated cost of 2 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +; Check select for a legal predicate vector +define @sel_nxv2i1() { +; CHECK-LABEL: 'sel_nxv2i1' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +define @sel_nxv4i1() { +; CHECK-LABEL: 'sel_nxv4i1' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +define @sel_nxv8i1() { +; CHECK-LABEL: 'sel_nxv8i1' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +define @sel_nxv16i1() { +; CHECK-LABEL: 'sel_nxv16i1' +; CHECK: Cost Model: Found an estimated cost of 1 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +; Check select for an illegal predicate vector +define @sel_nxv32i1() { +; CHECK-LABEL: 'sel_nxv32i1' +; CHECK: Cost Model: Found an estimated cost of 2 for instruction: %res = select undef, undef, undef +; CHECK: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = select undef, undef, undef + ret %res +} + +attributes #0 = { "target-features"="+sve,+bf16" } Index: llvm/test/Transforms/LoopVectorize/AArch64/sve-basic-vec.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LoopVectorize/AArch64/sve-basic-vec.ll @@ -0,0 +1,78 @@ +; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -mattr=+sve < %s -S 2>%t | FileCheck %s + +; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t + +; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it. +; WARN-NOT: warning + +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" +target triple = "aarch64-unknown-linux-gnu" + +define void @cmpsel_i32(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i64 %n) { +; CHECK-LABEL: @cmpsel_i32( +; CHECK-NEXT: entry: +; CHECK: vector.body: +; CHECK: [[WIDE_LOAD:%.*]] = load , * {{.*}}, align 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 0, i32 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP2:%.*]] = select [[TMP1]], shufflevector ( insertelement ( poison, i32 2, i32 0), poison, zeroinitializer), shufflevector ( insertelement ( poison, i32 10, i32 0), poison, zeroinitializer) +; CHECK: store [[TMP2]], * {{.*}}, align 4 +; +entry: + %cmp7 = icmp sgt i64 %n, 0 + br i1 %cmp7, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv + %0 = load i32, i32* %arrayidx, align 4 + %tobool.not = icmp eq i32 %0, 0 + %cond = select i1 %tobool.not, i32 2, i32 10 + %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv + store i32 %cond, i32* %arrayidx2, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond.not = icmp eq i64 %indvars.iv.next, %n + br i1 %exitcond.not, label %for.end.loopexit, label %for.body, !llvm.loop !0 + +for.end.loopexit: ; preds = %for.body + br label %for.end + +for.end: ; preds = %for.end.loopexit, %entry + ret void +} + +define void @cmpsel_f32(float* noalias nocapture %a, float* noalias nocapture readonly %b, i64 %n) { +; CHECK-LABEL: @cmpsel_f32( +; CHECK-NEXT: entry: +; CHECK: vector.body: +; CHECK: [[WIDE_LOAD:%.*]] = load , * {{.*}}, align 4 +; CHECK-NEXT: [[TMP1:%.*]] = fcmp ogt [[WIDE_LOAD]], shufflevector ( insertelement ( poison, float 3.000000e+00, i32 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP2:%.*]] = select [[TMP1]], shufflevector ( insertelement ( poison, float 1.000000e+01, i32 0), poison, zeroinitializer), shufflevector ( insertelement ( poison, float 2.000000e+00, i32 0), poison, zeroinitializer) +; CHECK: store [[TMP2]], * {{.*}}, align 4 + +entry: + %cmp8 = icmp sgt i64 %n, 0 + br i1 %cmp8, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv + %0 = load float, float* %arrayidx, align 4 + %cmp1 = fcmp ogt float %0, 3.000000e+00 + %conv = select i1 %cmp1, float 1.000000e+01, float 2.000000e+00 + %arrayidx3 = getelementptr inbounds float, float* %a, i64 %indvars.iv + store float %conv, float* %arrayidx3, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond.not = icmp eq i64 %indvars.iv.next, %n + br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !6 + +for.end: ; preds = %for.body, %entry + ret void +} + +!0 = distinct !{!0, !1, !2, !3, !4, !5} +!1 = !{!"llvm.loop.mustprogress"} +!2 = !{!"llvm.loop.vectorize.width", i32 4} +!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true} +!4 = !{!"llvm.loop.interleave.count", i32 1} +!5 = !{!"llvm.loop.vectorize.enable", i1 true} +!6 = distinct !{!6, !1, !2, !3, !4, !5}