diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -928,6 +928,10 @@ /// \return The width of the smallest vector register type. unsigned getMinVectorRegisterBitWidth() const; + /// \return The maximum value of vscale if the target specifies an + /// architectural maximum vector length, and None otherwise. + Optional getMaxVScale() const; + /// \return True if the vectorization factor should be chosen to /// make the vector of the smallest element type match the size of a /// vector register. For wider element types, this could result in @@ -1496,6 +1500,7 @@ virtual const char *getRegisterClassName(unsigned ClassID) const = 0; virtual unsigned getRegisterBitWidth(bool Vector) const = 0; virtual unsigned getMinVectorRegisterBitWidth() = 0; + virtual Optional getMaxVScale() const = 0; virtual bool shouldMaximizeVectorBandwidth(bool OptSize) const = 0; virtual unsigned getMinimumVF(unsigned ElemWidth) const = 0; virtual bool shouldConsiderAddressTypePromotion( @@ -1911,6 +1916,9 @@ unsigned getMinVectorRegisterBitWidth() override { return Impl.getMinVectorRegisterBitWidth(); } + Optional getMaxVScale() const override { + return Impl.getMaxVScale(); + } bool shouldMaximizeVectorBandwidth(bool OptSize) const override { return Impl.shouldMaximizeVectorBandwidth(OptSize); } diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -352,6 +352,8 @@ unsigned getMinVectorRegisterBitWidth() { return 128; } + Optional getMaxVScale() const { return None; } + bool shouldMaximizeVectorBandwidth(bool OptSize) const { return false; } unsigned getMinimumVF(unsigned ElemWidth) const { return 0; } diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -567,6 +567,8 @@ unsigned getRegisterBitWidth(bool Vector) const { return 32; } + Optional getMaxVScale() const { return None; } + /// Estimate the overhead of scalarizing an instruction. Insert and Extract /// are set if the demanded result elements need to be inserted and/or /// extracted from vectors. @@ -1246,14 +1248,12 @@ } } case Intrinsic::masked_gather: { - if (!isa(RetTy)) { - assert(VF.isScalar() && "Can't vectorize types here."); - const Value *Mask = Args[2]; - bool VarMask = !isa(Mask); - Align Alignment = cast(Args[1])->getAlignValue(); - return thisT()->getGatherScatterOpCost( - Instruction::Load, RetTy, Args[0], VarMask, Alignment, CostKind, I); - } + assert(VF.isScalar() && "Can't vectorize types here."); + const Value *Mask = Args[2]; + bool VarMask = !isa(Mask); + Align Alignment = cast(Args[1])->getAlignValue(); + return thisT()->getGatherScatterOpCost(Instruction::Load, RetTy, Args[0], + VarMask, Alignment, CostKind, I); } case Intrinsic::vector_reduce_add: case Intrinsic::vector_reduce_mul: diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -627,6 +627,10 @@ return TTIImpl->getMinVectorRegisterBitWidth(); } +Optional TargetTransformInfo::getMaxVScale() const { + return TTIImpl->getMaxVScale(); +} + bool TargetTransformInfo::shouldMaximizeVectorBandwidth(bool OptSize) const { return TTIImpl->shouldMaximizeVectorBandwidth(OptSize); } diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -115,8 +115,19 @@ return ST->getMinVectorRegisterBitWidth(); } + Optional getMaxVScale() const { + if (ST->hasSVE()) + return AArch64::SVEMaxBitsPerVector / AArch64::SVEBitsPerBlock; + return BaseT::getMaxVScale(); + } + unsigned getMaxInterleaveFactor(unsigned VF); + unsigned getGatherScatterOpCost(unsigned Opcode, Type *DataTy, + const Value *Ptr, bool VariableMask, + Align Alignment, TTI::TargetCostKind CostKind, + const Instruction *I = nullptr); + int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, const Instruction *I = nullptr); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -770,6 +770,27 @@ return Options; } +unsigned AArch64TTIImpl::getGatherScatterOpCost( + unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, + Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) { + auto LT = TLI->getTypeLegalizationCost(DL, DataTy); + ElementCount LVF = LT.second.getVectorElementCount(); + if (!LVF.isScalable()) + return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, + Alignment, CostKind, I); + + Optional MaxNumVScale = getMaxVScale(); + // TODO: Replace assert for InstructionCost + assert(MaxNumVScale && "Invalid Cost"); + + auto *VT = cast(DataTy); + unsigned MemOpCost = + getMemoryOpCost(Opcode, VT->getElementType(), Alignment, 0, CostKind, I); + // NumGather * (NumElementsPerGather) * MemOpCost + return LT.first * (MaxNumVScale.getValue() * LVF.getKnownMinValue()) * + MemOpCost; +} + bool AArch64TTIImpl::useNeonVector(const Type *Ty) const { return isa(Ty) && !ST->useSVEForFixedLengthVectors(); } diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-getIntrinsicInstrCost-gather.ll b/llvm/test/Analysis/CostModel/AArch64/sve-getIntrinsicInstrCost-gather.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Analysis/CostModel/AArch64/sve-getIntrinsicInstrCost-gather.ll @@ -0,0 +1,29 @@ +; Check getIntrinsicInstrCost in BasicTTIImpl.h for masked gather + +; RUN: opt -cost-model -analyze -mtriple=aarch64--linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s + +; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t + +; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it. +; WARN-NOT: warning + +define @masked_gather_nxv4i32( %ld, %masks, %passthru) { +; CHECK-LABEL: 'masked_gather_nxv4i32' +; CHECK-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %res = call @llvm.masked.gather.nxv4i32.nxv4p0i32( %ld, i32 0, %masks, %passthru) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret %res + %res = call @llvm.masked.gather.nxv4i32( %ld, i32 0, %masks, %passthru) + ret %res +} + + +define <4 x i32> @masked_gather_v4i32(<4 x i32*> %ld, <4 x i1> %masks, <4 x i32> %passthru) { +; CHECK-LABEL: 'masked_gather_v4i32' +; CHECK-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res + + %res = call <4 x i32> @llvm.masked.gather.v4i32(<4 x i32*> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru) + ret <4 x i32> %res +} + +declare @llvm.masked.gather.nxv4i32( %ptrs, i32 %align, %masks, %passthru) +declare <4 x i32> @llvm.masked.gather.v4i32(<4 x i32*> %ptrs, i32 %align, <4 x i1> %masks, <4 x i32> %passthru) diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-getIntrinsicInstrCost-scatter.ll b/llvm/test/Analysis/CostModel/AArch64/sve-getIntrinsicInstrCost-scatter.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Analysis/CostModel/AArch64/sve-getIntrinsicInstrCost-scatter.ll @@ -0,0 +1,31 @@ +; Check getIntrinsicInstrCost in BasicTTIImpl.h with for masked scatter + +; RUN: opt -cost-model -analyze -mtriple=aarch64--linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s + +; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t + +; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it. +; WARN-NOT: warning + + +define void @masked_scatter_nxv4i32( %data, %ptrs, %masks) { +; CHECK-LABEL: 'masked_scatter_nxv4i32' +; CHECK-NEXT: Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0i32( %data, %ptrs, i32 0, %masks) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void + + call void @llvm.masked.scatter.nxv4i32( %data, %ptrs, i32 0, %masks) + ret void +} + + +define void @masked_scatter_v4i32(<4 x i32> %data, <4 x i32*> %ptrs, <4 x i1> %masks) { +; CHECK-LABEL: 'masked_scatter_v4i32' +; CHECK-NEXT: Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %data, <4 x i32*> %ptrs, i32 0, <4 x i1> %masks) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void + + call void @llvm.masked.scatter.v4i32(<4 x i32> %data, <4 x i32*> %ptrs, i32 0, <4 x i1> %masks) + ret void +} + +declare void @llvm.masked.scatter.nxv4i32( %data, %ptrs, i32 %align, %masks) +declare void @llvm.masked.scatter.v4i32(<4 x i32> %data, <4 x i32*> %ptrs, i32 %align, <4 x i1> %masks)